tests: add offline local test server prototype (cookies + HTTPS)

Replace the network dependency for crawl tests with a self-contained Python
stdlib server (http.server + ssl) that httrack crawls over loopback. The server
binds an ephemeral port and prints it on stdout; local-crawl.sh discovers the
port, substitutes the BASEURL token into the httrack arguments, runs the crawl,
and audits the mirror under the discovered host-root directory.

This prototype migrates two cases off ut.httrack.com:

- 13_local-cookies.test drives the cookie chain (entrance/second/third)
  reimplemented as Python handlers from the old ut/cookies/*.php fixtures. A
  missing or wrong cookie answers 500, so a clean 3-files/0-errors run proves
  the cookie jar is replayed across links.
- 14_local-https.test crawls over HTTPS using a shipped long-dated self-signed
  cert. httrack does not verify certs, so the cert is accepted as-is and the
  real TLS path runs offline.

The group skips (exit 77) when python3 is missing, mirroring check-network.sh.
Fixtures and the cert are listed explicitly in EXTRA_DIST (automake does not
expand globs); make distcheck passes.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+# Match black's formatting so the two tools don't fight.
+max-line-length = 88
+# E203/W503 conflict with black's slice and line-break style.
+extend-ignore = E203, W503

.github/workflows/ci.yml

@@ -171,8 +171,16 @@ jobs:
         # Leaks at exit are out of scope (the CLI frees little on the way out);
         # we want memory-safety errors, so turn leak detection off and make every
         # other finding abort the run.
+        #
+        # Poison fresh allocations with 0xCA and freed blocks with 0xCB (decimal
+        # 202/203) so memory never reads back as accidental zeros: a missing-NUL
+        # fread buffer then runs strlen off into the redzone instead of stopping
+        # at a lucky zero. Distinct bytes tell the two apart in a dump (0xCA =
+        # uninitialized, 0xCB = use-after-free). ASan caps its malloc fill at 4096
+        # bytes by default, so max_malloc_fill_size lifts it to cover large cache
+        # buffers; free_fill flags use-after-free reads.
         env:
-          ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1
+          ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
         run: make check
 

src/htscache.c

@@ -939,7 +939,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                   FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
 
                   if (fp != NULL) {
-                    r.adr = (char *) malloct((int) r.size + 4);
+                    r.adr = (char *) malloct((int) r.size + 1);
                     if (r.adr != NULL) {
                       if (r.size > 0
                           && fread(r.adr, 1, (int) r.size, fp) != r.size) {
@@ -966,7 +966,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
               // Data in cache.
               else {
                 // lire fichier (d'un coup)
-                r.adr = (char *) malloct((int) r.size + 4);
+                r.adr = (char *) malloct((int) r.size + 1);
                 if (r.adr != NULL) {
                   if (unzReadCurrentFile((unzFile) cache->zipInput, r.adr, (int) r.size) != r.size) {   // erreur
                     freet(r.adr);
@@ -1246,7 +1246,7 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                     FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), return_save), "rb");
 
                     if (fp != NULL) {
-                      r.adr = (char *) malloct((size_t) r.size + 4);
+                      r.adr = (char *) malloct((size_t) r.size + 1);
                       if (r.adr != NULL) {
                         if (r.size > 0
                             && fread(r.adr, 1, (size_t) r.size, fp) != r.size) {
@@ -1268,7 +1268,7 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                 }
               } else {
                 // lire fichier (d'un coup)
-                r.adr = (char *) malloct((size_t) r.size + 4);
+                r.adr = (char *) malloct((size_t) r.size + 1);
                 if (r.adr != NULL) {
                   if (fread(r.adr, 1, (size_t) r.size, cache->olddat) != r.size) {      // erreur
                     freet(r.adr);
@@ -1371,7 +1371,7 @@ int cache_readdata(cache_back * cache, const char *str1, const char *str2,
 
         cache_rint(cache->olddat, &len);
         if (len > 0) {
-          char *mem_buff = (char *) malloct(len + 4);   /* Plus byte 0 */
+          char *mem_buff = (char *) malloct(len + 1); /* trailing \0 */
 
           if (mem_buff) {
             if (fread(mem_buff, 1, len, cache->olddat) == len) {        // lire tout (y compris statuscode etc)*/

src/htscore.c

@@ -633,13 +633,12 @@ int httpmirror(char *url1, httrackp * opt) {
     // c'est plus propre et plus logique que d'entrer à la main les liens dans la pile
     // on bénéficie ainsi des vérifications et des tests du robot pour les liens "primaires"
     primary = (char *) malloct(primary_len);
-    if (primary) {
-      primary[0] = '\0';
-    } else {
+    if (!primary) {
       printf("PANIC! : Not enough memory [%d]\n", __LINE__);
       XH_extuninit;
       return 0;
     }
+    htsbuff primarybuff = htsbuff_ptr(primary, primary_len);
 
     while(*a) {
       int i;
@@ -687,11 +686,11 @@ int httpmirror(char *url1, httrackp * opt) {
               strcatbuff(tempo, "*");   // ajouter un *
             }
           }
-          if (type)
-            strcpybuff(filters[filptr], "+");
-          else
-            strcpybuff(filters[filptr], "-");
-          strcatbuff(filters[filptr], tempo);
+          {
+            htsbuff fb = htsbuff_ptr(filters[filptr], HTS_URLMAXSIZE * 2);
+            htsbuff_cpy(&fb, type ? "+" : "-");
+            htsbuff_cat(&fb, tempo);
+          }
           filptr++;
 
           /* sanity check */
@@ -726,12 +725,10 @@ int httpmirror(char *url1, httrackp * opt) {
         }
         url[i++] = '\0';
 
-        //strcatbuff(primary,"<PRIMARY=\"");
         if (strstr(url, ":/") == NULL)
-          strcatbuff(primary, "http://");
-        strcatbuff(primary, url);
-        //strcatbuff(primary,"\">");
-        strcatbuff(primary, "\n");
+          htsbuff_cat(&primarybuff, "http://");
+        htsbuff_cat(&primarybuff, url);
+        htsbuff_cat(&primarybuff, "\n");
       }
     }                           // while
 
@@ -762,7 +759,6 @@ int httpmirror(char *url1, httrackp * opt) {
         int filelist_ptr = 0;
         int n = 0;
         char BIGSTK line[HTS_URLMAXSIZE * 2];
-        char *primary_ptr = primary + strlen(primary);
 
         while(filelist_ptr < filelist_sz) {
           int count =
@@ -771,13 +767,10 @@ int httpmirror(char *url1, httrackp * opt) {
           if (count && line[0]) {
             n++;
             if (strstr(line, ":/") == NULL) {
-              strcpybuff(primary_ptr, "http://");
-              primary_ptr += strlen(primary_ptr);
+              htsbuff_cat(&primarybuff, "http://");
             }
-            strcpybuff(primary_ptr, line);
-            primary_ptr += strlen(primary_ptr);
-            strcpybuff(primary_ptr, "\n");
-            primary_ptr += 1;
+            htsbuff_cat(&primarybuff, line);
+            htsbuff_cat(&primarybuff, "\n");
           }
         }
         // fclose(fp);
@@ -2453,9 +2446,10 @@ void host_ban(httrackp * opt, int ptr,
   // interdire host
   assertf((*_FILTERS_PTR) < opt->maxfilter);
   if (*_FILTERS_PTR < opt->maxfilter) {
-    strcpybuff(_FILTERS[*_FILTERS_PTR], "-");
-    strcatbuff(_FILTERS[*_FILTERS_PTR], host);
-    strcatbuff(_FILTERS[*_FILTERS_PTR], "/*");  // host/ * interdit
+    htsbuff fb = htsbuff_ptr(_FILTERS[*_FILTERS_PTR], HTS_URLMAXSIZE * 2);
+    htsbuff_cpy(&fb, "-");
+    htsbuff_cat(&fb, host);
+    htsbuff_cat(&fb, "/*"); // forbid host/*
     (*_FILTERS_PTR)++;
   }
   // oups
@@ -3518,7 +3512,7 @@ char *next_token(char *p, int flag) {
   p--;
   do {
     p++;
-    if (flag && (*p == '\\')) { // sauter \x ou \"
+    if (flag && (*p == '\\')) { // skip \x or \"
       if (quote) {
         char c = '\0';
 
@@ -3527,20 +3521,14 @@ char *next_token(char *p, int flag) {
         else if (*(p + 1) == '"')
           c = '"';
         if (c) {
-          char BIGSTK tempo[8192];
-
-          tempo[0] = c;
-          tempo[1] = '\0';
-          strcatbuff(tempo, p + 2);
-          strcpybuff(p, tempo);
+          /* unescape the 2 chars to one, shifting left in place */
+          *p = c;
+          memmove(p + 1, p + 2, strlen(p + 2) + 1);
         }
       }
-    } else if (*p == 34) {      // guillemets (de fin)
-      char BIGSTK tempo[8192];
-
-      tempo[0] = '\0';
-      strcatbuff(tempo, p + 1);
-      strcpybuff(p, tempo);     /* wipe "" */
+    } else if (*p == 34) { // closing quote
+      /* drop the quote, shifting the rest left in place */
+      memmove(p, p + 1, strlen(p + 1) + 1);
       p--;
       /* */
       quote = !quote;
@@ -3880,7 +3868,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
                                 afs.af.adr, afs.save, savename(), tempo);
                   if (str->localLink
                       && str->localLinkSize > (int) strlen(tempo) + 1) {
-                    strcpybuff(str->localLink, tempo);
+                    strlcpybuff(str->localLink, tempo, str->localLinkSize);
                   }
                 }
               }
@@ -3892,11 +3880,11 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
                           lien);
             if (str->localLink
                 && str->localLinkSize > (int) (strlen(afs.af.adr) + strlen(afs.af.fil) + 8)) {
-              str->localLink[0] = '\0';
+              htsbuff lb = htsbuff_ptr(str->localLink, str->localLinkSize);
               if (!link_has_authority(afs.af.adr))
-                strcpybuff(str->localLink, "http://");
-              strcatbuff(str->localLink, afs.af.adr);
-              strcatbuff(str->localLink, afs.af.fil);
+                htsbuff_cat(&lb, "http://");
+              htsbuff_cat(&lb, afs.af.adr);
+              htsbuff_cat(&lb, afs.af.fil);
             }
             r = -1;
           }

src/htscoremain.c

@@ -236,6 +236,55 @@ static void basic_selftests(void) {
     }
     freet(slots);
   }
+  // next_token(): in-place token scanner. Strips surrounding quotes, unescapes
+  // \" and \\ when flag is set, and returns the token terminator (the space, or
+  // NULL at end of string). The unquote/unescape rewrites the string in place
+  // by shifting left, so the result is always shorter -- regression for that
+  // compaction.
+  {
+    char tok[64];
+
+    // plain token: unchanged, returns a pointer AT the separating space (exact
+    // position, not just any space -- a strchr-style impl would land elsewhere
+    // once quotes shift the content)
+    strcpybuff(tok, "abc def");
+    {
+      char *const end = next_token(tok, 0);
+      assertf(end == tok + 3 && *end == ' ' && strcmp(tok, "abc def") == 0);
+    }
+    // surrounding quotes stripped, returns the (post-shift) trailing space
+    strcpybuff(tok, "\"ab\" cd");
+    {
+      char *const end = next_token(tok, 1);
+      assertf(end == tok + 2 && *end == ' ' && strcmp(tok, "ab cd") == 0);
+    }
+    // a space inside quotes does not end the token; end of string returns NULL
+    strcpybuff(tok, "\"a b\"c");
+    {
+      char *const end = next_token(tok, 1);
+      assertf(end == NULL && strcmp(tok, "a bc") == 0);
+    }
+    // \" and \\ are unescaped to literal " and \ in place
+    strcpybuff(tok, "\"a\\\"b\\\\c\"");
+    {
+      char *const end = next_token(tok, 1);
+      assertf(end == NULL && strcmp(tok, "a\"b\\c") == 0);
+    }
+    // unterminated quote: the opening quote is dropped, the rest survives, and
+    // the scan runs to the NUL (returns NULL)
+    strcpybuff(tok, "\"ab");
+    {
+      char *const end = next_token(tok, 1);
+      assertf(end == NULL && strcmp(tok, "ab") == 0);
+    }
+    // trailing lone backslash in a quote: *(p+1) is the NUL, not an escape, so
+    // the backslash is kept intact (and there is no over-read past the NUL)
+    strcpybuff(tok, "\"a\\");
+    {
+      char *const end = next_token(tok, 1);
+      assertf(end == NULL && strcmp(tok, "a\\") == 0);
+    }
+  }
 }
 
 /* Self-tests for the htssafe.h bounded string ops (driven by httrack -#8).

src/htsindex.c

@@ -334,7 +334,7 @@ void index_finish(const char *indexpath, int mode) {
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
-        blk = malloct(size + 4);
+        blk = malloct(size + 1);
         if (blk) {
           fseek(fp_tmpproject, 0, SEEK_SET);
           if ((INTsys) fread(blk, 1, size, fp_tmpproject) == size) {

src/proxy/store.c

@@ -1162,7 +1162,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                     FILE *fp = fopen(file_convert(catbuff, sizeof(catbuff), previous_save), "rb");
 
                     if (fp != NULL) {
-                      r->adr = (char *) malloc(r->size + 4);
+                      r->adr = (char *) malloc(r->size + 1);
                       if (r->adr != NULL) {
                         if (r->size > 0
                             && fread(r->adr, 1, r->size, fp) != r->size) {
@@ -1172,6 +1172,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                           sprintf(r->msg, "Read error in cache disk data: %s",
                                   strerror(last_errno));
                         }
+                        r->adr[r->size] = '\0';
                       } else {
                         r->statuscode = STATUSCODE_INVALID;
                         strcpy(r->msg,

tests/13_local-cookies.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Cookie chain against the local test server (replaces the old online
+# ut/cookies/*.php fixtures). entrance.php sets cat/cake; second.php checks
+# them and sets badger; third.php checks all three. A missing or wrong cookie
+# returns 500, which would surface as an httrack error and a missing file, so a
+# clean 3-files/0-errors run proves the cookie jar is replayed across links.
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --files 3 \
+    --found 'cookies/entrance.html' \
+    --found 'cookies/second.html' \
+    --found 'cookies/third.html' \
+    httrack 'BASEURL/cookies/entrance.php'

tests/14_local-https.test

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# HTTPS crawl against the local test server, using the shipped self-signed
+# cert. httrack does not verify certs (htslib.c: SSL_CTX_new with no
+# SSL_CTX_set_verify), so the self-signed cert is accepted as-is and this
+# exercises the real TLS path offline. basic.html links to link.html with four
+# distinct query strings, each saved under a hashed name -> 5 files.
+
+: "${top_srcdir:=..}"
+
+if test "$HTTPS_SUPPORT" == "no"; then
+    echo "no https support compiled, skipping"
+    exit 77
+fi
+
+bash "$top_srcdir/tests/local-crawl.sh" --tls --errors 0 --files 5 \
+    --found 'simple/basic.html' \
+    httrack 'BASEURL/simple/basic.html'

tests/Makefile.am

@@ -1,4 +1,7 @@
-EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh
+# Note: EXTRA_DIST globs are NOT expanded by automake; list fixtures explicitly.
+EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
+	local-crawl.sh local-server.py server.crt server.key \
+	server-root/simple/basic.html server-root/simple/link.html
 
 TESTS_ENVIRONMENT =
 TESTS_ENVIRONMENT += PATH=$(top_builddir)/src$(PATH_SEPARATOR)$$PATH
@@ -35,6 +38,8 @@ TESTS = \
 	11_crawl-international.test \
 	11_crawl-longurl.test \
 	11_crawl-parsing.test \
-	12_crawl_https.test
+	12_crawl_https.test \
+	13_local-cookies.test \
+	14_local-https.test
 
 CLEANFILES = check-network_sh.cache

tests/Makefile.in

@@ -476,7 +476,12 @@ target_alias = @target_alias@
 top_build_prefix = @top_build_prefix@
 top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
-EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh
+
+# Note: EXTRA_DIST globs are NOT expanded by automake; list fixtures explicitly.
+EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
+	local-crawl.sh local-server.py server.crt server.key \
+	server-root/simple/basic.html server-root/simple/link.html
+
 # note: libtool should handle that
 ### TESTS_ENVIRONMENT += $(SHLIBPATH_VAR)="$(top_builddir)/src/$(LT_CV_OBJDIR)$${$(SHLIBPATH_VAR):+$(PATH_SEPARATOR)}$$$(SHLIBPATH_VAR)"
 TESTS_ENVIRONMENT = PATH=$(top_builddir)/src$(PATH_SEPARATOR)$$PATH \
@@ -509,7 +514,9 @@ TESTS = \
 	11_crawl-international.test \
 	11_crawl-longurl.test \
 	11_crawl-parsing.test \
-	12_crawl_https.test
+	12_crawl_https.test \
+	13_local-cookies.test \
+	14_local-https.test
 
 CLEANFILES = check-network_sh.cache
 all: all-am

tests/local-crawl.sh

@@ -0,0 +1,235 @@
+#!/bin/bash
+#
+# Launcher for httrack crawl tests against the local Python test server.
+#
+# Starts tests/local-server.py on an ephemeral port, discovers the port from
+# the server's stdout, then runs httrack against http(s)://127.0.0.1:$PORT and
+# audits the mirror. The server is always killed and the tmpdir removed on exit.
+#
+# The token BASEURL in any httrack argument is replaced with the discovered
+# http(s)://127.0.0.1:$PORT base. --found/--directory paths are relative to the
+# discovered host root (127.0.0.1_<port>/), since the random port leaks into
+# the mirror directory name.
+#
+# Usage:
+#   bash local-crawl.sh [--tls] [--root DIR] \
+#       --errors N --files N --found PATH ... --directory PATH ... \
+#       httrack BASEURL/some/path [httrack-args...]
+
+set -u
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+server="${testdir}/local-server.py"
+root="${LOCAL_SERVER_ROOT:-${testdir}/server-root}"
+cert="${testdir}/server.crt"
+key="${testdir}/server.key"
+
+tls=
+verbose=
+tmpdir=
+serverpid=
+crawlpid=
+
+function warning {
+    echo "** $*" >&2
+    return 0
+}
+function die {
+    warning "$*"
+    exit 1
+}
+function debug {
+    test -n "$verbose" && echo "$*" >&2
+    return 0
+}
+function info { printf "[%s] ..\t" "$*" >&2; }
+function result { echo "$*" >&2; }
+
+function cleanup {
+    if test -n "$crawlpid"; then
+        kill -9 "$crawlpid" 2>/dev/null
+        crawlpid=
+    fi
+    if test -n "$serverpid"; then
+        kill "$serverpid" 2>/dev/null
+        # Reap it so the port is released before we rm the tmpdir/log.
+        wait "$serverpid" 2>/dev/null
+        serverpid=
+    fi
+    if test -n "$tmpdir" && test -d "$tmpdir"; then
+        test -n "$nopurge" || rm -rf "$tmpdir"
+    fi
+}
+
+function assert_equals {
+    info "$1"
+    if test ! "$2" == "$3"; then
+        result "expected '$2', got '$3'"
+        exit 1
+    fi
+    result "OK ($2)"
+}
+
+nopurge=
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+# python3 is required; mirror check-network.sh's skip-with-77 convention.
+command -v python3 >/dev/null || ! echo "python3 not found; skipping local crawl tests" || exit 77
+
+tmptopdir=${TMPDIR:-/tmp}
+test -d "$tmptopdir" || mkdir -p "$tmptopdir" || die "no temporary directory; set TMPDIR"
+tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create tmpdir"
+
+# --- parse leading control flags --------------------------------------------
+declare -a audit=()
+scheme=http
+pos=0
+args=("$@")
+nargs=$#
+while test "$pos" -lt "$nargs"; do
+    case "${args[$pos]}" in
+    --debug) verbose=1 ;;
+    --no-purge)
+        nopurge=1
+        audit+=("--no-purge")
+        ;;
+    --tls)
+        tls=1
+        scheme=https
+        ;;
+    --root)
+        pos=$((pos + 1))
+        root="${args[$pos]}"
+        ;;
+    --errors | --files)
+        audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
+        pos=$((pos + 1))
+        ;;
+    --found | --not-found | --directory)
+        audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
+        pos=$((pos + 1))
+        ;;
+    httrack)
+        pos=$((pos + 1))
+        break
+        ;;
+    *) die "unrecognized option ${args[$pos]}" ;;
+    esac
+    pos=$((pos + 1))
+done
+
+# --- start the server --------------------------------------------------------
+test -r "$server" || die "cannot read $server"
+serverlog="${tmpdir}/server.log"
+serverargs=(--root "$root")
+if test -n "$tls"; then
+    serverargs+=(--tls --cert "$cert" --key "$key")
+fi
+debug "starting python3 $server ${serverargs[*]}"
+python3 "$server" "${serverargs[@]}" >"$serverlog" 2>&1 &
+serverpid=$!
+
+# Wait for the "PORT <n>" line (server prints it once bound).
+port=
+for _ in $(seq 1 50); do
+    if test -s "$serverlog"; then
+        line=$(head -n1 "$serverlog")
+        if test "${line%% *}" == "PORT"; then
+            port="${line#PORT }"
+            break
+        fi
+    fi
+    kill -0 "$serverpid" 2>/dev/null || die "server exited early: $(cat "$serverlog")"
+    sleep 0.1
+done
+test -n "$port" || die "could not discover server port: $(cat "$serverlog")"
+debug "server listening on ${scheme}://127.0.0.1:${port}"
+
+baseurl="${scheme}://127.0.0.1:${port}"
+
+# --- substitute BASEURL in the remaining (httrack) args ----------------------
+declare -a hts=()
+while test "$pos" -lt "$nargs"; do
+    hts+=("${args[$pos]//BASEURL/$baseurl}")
+    pos=$((pos + 1))
+done
+
+# --- run httrack -------------------------------------------------------------
+which httrack >/dev/null || die "could not find httrack"
+ver=$(httrack -O /dev/null --version | sed -e 's/HTTrack version //')
+test -n "$ver" || die "could not run httrack"
+
+out="${tmpdir}/crawl"
+mkdir "$out" || die "could not create $out"
+# Localhost is fast; disable the rate/bandwidth safety limits but keep a
+# max-time backstop so a hang cannot wedge the suite.
+declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
+log="${tmpdir}/log"
+info "running httrack ${hts[*]}"
+httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+crawlpid=$!
+wait "$crawlpid"
+crawlres=$?
+crawlpid=
+# httrack exits 0 even on hard connect/DNS errors, so this is a backstop only;
+# the real guard is the audit below (--errors 0 plus the host-root existence check).
+test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
+    cat "$log" >&2
+    exit 1
+}
+result "OK"
+grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
+
+# --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
+hostroot=
+for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
+    if test -d "$cand"; then
+        hostroot="$cand"
+        break
+    fi
+done
+test -n "$hostroot" || die "could not find host root under $out"
+debug "host root: $hostroot"
+
+# --- audit -------------------------------------------------------------------
+i=0
+while test "$i" -lt "${#audit[@]}"; do
+    case "${audit[$i]}" in
+    --errors)
+        i=$((i + 1))
+        assert_equals "checking errors" "${audit[$i]}" \
+            "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
+        ;;
+    --files)
+        i=$((i + 1))
+        nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
+            sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
+        assert_equals "checking files" "${audit[$i]}" "$nFiles"
+        ;;
+    --found)
+        i=$((i + 1))
+        info "checking for ${audit[$i]}"
+        if test -f "${hostroot}/${audit[$i]}"; then result "OK"; else
+            result "not found"
+            exit 1
+        fi
+        ;;
+    --not-found)
+        i=$((i + 1))
+        info "checking absence of ${audit[$i]}"
+        if test ! -f "${hostroot}/${audit[$i]}"; then result "OK"; else
+            result "present"
+            exit 1
+        fi
+        ;;
+    --directory)
+        i=$((i + 1))
+        info "checking for dir ${audit[$i]}"
+        if test -d "${hostroot}/${audit[$i]}"; then result "OK"; else
+            result "not found"
+            exit 1
+        fi
+        ;;
+    esac
+    i=$((i + 1))
+done

tests/local-server.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Self-contained local web server for httrack's crawl tests.
+
+Serves static fixtures from a docroot plus a handful of dynamic endpoints
+(cookies, ...) so httrack can be exercised over loopback, deterministically and
+offline, instead of crawling the live ut.httrack.com.
+
+Binds to an ephemeral port (port 0) and prints the chosen port to stdout as
+"PORT <n>\n" so a launcher can discover it. Pass --tls to wrap the socket with
+the shipped self-signed test cert; httrack does not verify certs, so no CA
+trust plumbing is needed.
+
+stdlib only (http.server + ssl) -- no new build or runtime dependency.
+"""
+
+import argparse
+import os
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import quote, unquote, urlsplit
+
+# Cookie chain replicated from the old ut/cookies/*.php fixtures.
+COOKIE_PATH = "/cookies/"
+COOKIES = {
+    "cat": "dog",
+    "cake": "is a lie!",
+    "badger": "mushroom, with 'ants'",
+}
+
+PAGE = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+\t"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+<head>
+\t<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+\t<title>Sample test</title>
+</head>
+<body>
+{body}
+</body>
+</html>
+"""
+
+
+class Handler(SimpleHTTPRequestHandler):
+    # Quieter logging; the launcher captures httrack's own log anyway.
+    def log_message(self, fmt, *args):
+        if os.environ.get("LOCAL_SERVER_VERBOSE"):
+            super().log_message(fmt, *args)
+
+    # --- helpers -----------------------------------------------------------
+
+    def request_cookies(self):
+        """Parse the Cookie header into {name: decoded-value}.
+
+        Mirrors PHP's $_COOKIE: values are url-decoded, matching the encoding
+        applied when the cookie was set (see set_cookie)."""
+        jar = {}
+        raw = self.headers.get("Cookie", "")
+        for pair in raw.split(";"):
+            pair = pair.strip()
+            if "=" in pair:
+                name, value = pair.split("=", 1)
+                jar[name.strip()] = unquote(value.strip())
+        return jar
+
+    def set_cookie(self, name, value):
+        """Queue a Set-Cookie header, url-encoding the value like PHP's
+        setcookie() so spaces/quotes/commas stay a single token that httrack
+        can store and replay verbatim."""
+        self._set_cookies.append(f"{name}={quote(value)}; Path={COOKIE_PATH}")
+
+    def send_html(self, body, status=200, extra_status=None):
+        encoded = PAGE.format(body=body).encode("utf-8")
+        self.send_response(status, extra_status)
+        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.send_header("Content-Length", str(len(encoded)))
+        for cookie in self._set_cookies:
+            self.send_header("Set-Cookie", cookie)
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(encoded)
+
+    def fail_cookie(self, what):
+        # The old PHPs answered 500 with the reason in the status line.
+        self.send_html("", status=500, extra_status=f"The {what} is missing or invalid")
+
+    # --- dynamic routes ----------------------------------------------------
+
+    def route_entrance(self):
+        self.set_cookie("cat", COOKIES["cat"])
+        self.set_cookie("cake", COOKIES["cake"])
+        self.send_html('\tThis is a <a href="second.php">link</a>')
+
+    def route_second(self):
+        jar = self.request_cookies()
+        if jar.get("cat") != COOKIES["cat"]:
+            return self.fail_cookie("cat")
+        if jar.get("cake") != COOKIES["cake"]:
+            return self.fail_cookie("cake")
+        self.set_cookie("badger", COOKIES["badger"])
+        self.send_html('\tThis is a <a href="third.php">link</a>')
+
+    def route_third(self):
+        jar = self.request_cookies()
+        if jar.get("cat") != COOKIES["cat"]:
+            return self.fail_cookie("cat")
+        if jar.get("cake") != COOKIES["cake"]:
+            return self.fail_cookie("cake")
+        if jar.get("badger") != COOKIES["badger"]:
+            return self.fail_cookie("badger")
+        self.send_html("\tThis is a test.")
+
+    def route_robots(self):
+        body = b"User-agent: *\nDisallow:\n"
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(body)
+
+    ROUTES = {
+        "/cookies/entrance.php": route_entrance,
+        "/cookies/second.php": route_second,
+        "/cookies/third.php": route_third,
+        "/robots.txt": route_robots,
+    }
+
+    # --- dispatch ----------------------------------------------------------
+
+    def dispatch(self):
+        self._set_cookies = []
+        path = urlsplit(self.path).path
+        handler = self.ROUTES.get(path)
+        if handler is not None:
+            handler(self)
+            return True
+        return False
+
+    def do_GET(self):
+        if not self.dispatch():
+            super().do_GET()
+
+    def do_HEAD(self):
+        if not self.dispatch():
+            super().do_HEAD()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", required=True, help="docroot for static files")
+    parser.add_argument("--bind", default="127.0.0.1", help="bind address")
+    parser.add_argument("--tls", action="store_true", help="serve HTTPS")
+    parser.add_argument("--cert", help="TLS certificate (PEM)")
+    parser.add_argument("--key", help="TLS private key (PEM)")
+    args = parser.parse_args()
+
+    root = os.path.abspath(args.root)
+
+    def factory(*a, **kw):
+        return Handler(*a, directory=root, **kw)
+
+    httpd = ThreadingHTTPServer((args.bind, 0), factory)
+
+    if args.tls:
+        import ssl
+
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        ctx.load_cert_chain(certfile=args.cert, keyfile=args.key)
+        httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
+
+    port = httpd.socket.getsockname()[1]
+    # The launcher reads this line to discover the ephemeral port.
+    print(f"PORT {port}", flush=True)
+
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    main()

tests/server-root/simple/basic.html

@@ -0,0 +1,18 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+	"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" lang="fr">
+
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	<title>Sample test</title>
+</head>
+
+<body>
+
+	This is a <a href="link.html?v=1">link</a>
+	This is a <a href='link.html?v=2'>link</a>
+	This is a <a href="./link.html?v=3">link</a>
+	This is a <a href=link.html?v=4>link</a>
+
+</body>

tests/server-root/simple/link.html

@@ -0,0 +1,3 @@
+This is a link.
+
+Go back to <a href="basic.html">home</a>.

tests/server.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDbzCCAlegAwIBAgIUdWkDDomnY3WW95UqJ+UOASuR/i0wDQYJKoZIhvcNAQEL
+BQAwODESMBAGA1UEAwwJMTI3LjAuMC4xMSIwIAYDVQQKDBlIVFRyYWNrIGxvY2Fs
+IHRlc3Qgc2VydmVyMCAXDTI2MDYxNTE0NDQxMFoYDzIwNTYwNjA3MTQ0NDEwWjA4
+MRIwEAYDVQQDDAkxMjcuMC4wLjExIjAgBgNVBAoMGUhUVHJhY2sgbG9jYWwgdGVz
+dCBzZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDx78mogNhT
+noWwRa51NeGtapQ1PfTYLlIMUzuloFXOsR1/ozRkFucqHNftF22wf0gg4VQJSBSf
+3rwj79vsnt3nyaD03bTAafpHXkd+IJxQowiG8TfOJF0R/Qg9g7DCE66R9agQpMJC
+SGxIin9p/4ld4Hn6869d4hNq4fHxNf/qkj2cnf8DYxrldz2FGsi6yMed4tzz2Am4
+ZbPgwep+fy843ZdYrVIms9vJluNa9E+6Vpw9FwdjzQ/IBBMLvGaC2pDkc95YelaE
+nQrAlTO/0l5vjc8XuTQFlo3DbUg+WEld/pxvCqsd/q1mqjL0WbxtXl2zCwGzAoJx
+rjVEPfA8QSbtAgMBAAGjbzBtMB0GA1UdDgQWBBTHE0KKW8REV4HxajzVsIBxz3iL
+9zAfBgNVHSMEGDAWgBTHE0KKW8REV4HxajzVsIBxz3iL9zAPBgNVHRMBAf8EBTAD
+AQH/MBoGA1UdEQQTMBGHBH8AAAGCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOC
+AQEAYlTEftrwGJBXuPmtxhmtw2HO/VTC4TGnq67hH5H+ptwgZJuuxCQ5KW6flTyp
+FTyMhha33WD4EBL3wqqJsWr9Y4BXqi4G0lRqXBcC1oIUa2VYIDMER7kaY1qTSqE8
+ARpwdB2BhvngAzDLc+4Jt4jQMRGr8fHAwxpDBoIZ1knbyzYNP73Bajse6/8YtxUu
+nB2BsldjZnLvyHvRxUpWp92OyQih4jYSrlN6olDFlKDg7++kMhkHtJQW9a1t54VN
+0ZXrB1ZRuHUUvGBq26x71riTWor7HNOSQaGeCMQjZNQkh5tfshNygUGSZVXTEwhG
+xSrOL7NqBt2+EkVwf7LjGzjmBw==
+-----END CERTIFICATE-----

tests/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDx78mogNhTnoWw
+Ra51NeGtapQ1PfTYLlIMUzuloFXOsR1/ozRkFucqHNftF22wf0gg4VQJSBSf3rwj
+79vsnt3nyaD03bTAafpHXkd+IJxQowiG8TfOJF0R/Qg9g7DCE66R9agQpMJCSGxI
+in9p/4ld4Hn6869d4hNq4fHxNf/qkj2cnf8DYxrldz2FGsi6yMed4tzz2Am4ZbPg
+wep+fy843ZdYrVIms9vJluNa9E+6Vpw9FwdjzQ/IBBMLvGaC2pDkc95YelaEnQrA
+lTO/0l5vjc8XuTQFlo3DbUg+WEld/pxvCqsd/q1mqjL0WbxtXl2zCwGzAoJxrjVE
+PfA8QSbtAgMBAAECggEACgNK4klq1T3IpKdNoBY5yoE7CbUQZBNkBpSPRxHgBezj
+SVFfgrZGnOySrIJSt4JHtuynG2Hl+0ku74HRep/ck+eOsh5W3mZvGvMLnGxhwR3u
+Or99osTIgU0VQTkpC0SLQ16FCnih0uJycNIikdLR7uuya1tt1OyIBzK7XlNGIywT
+p85zJc7/6TfTC9eM7lqh7JGR7KplBxSvgZL1pUr7y4rNpKms6uzOvPND79CcKnbU
+BBA9Tu4qdOkoOljsZKkvh3pihxyG9X6d8QTZ/uX3pkvliwSFBc+Sz9EootA3/4r5
+gVWpQ2t/AY7fY4hqzLIX/HivVaPj3cWk1G+SHm0XNQKBgQD5I9rijqFvV/p6FmUl
+FbnjJFFHHgZLivlGxAC5vOyJNQQaqdeDzg7yMotNmQTggVGjT6sjdosQb3n+ctuk
+EhQnZSU5VkNKv1+PTR35WrRkaECCaqz3Pv79pV9GVcX3it7UuYjNiOeSPqINWe+X
+49JwnJFz+qQ1BchAwOis4zkENwKBgQD4mShDaYLOO97VpgZj4cGxHHWyEK9CRQvp
+I7HxRmfaWS3JHwb88lOmALEU6pAj5cYJPAznv8BnUWcVHalZbkQ1JWYtUJRqj6OI
+Ym7rw/nm4Ay5ijbdEism173dSk3IjOe+PdAlxzsOuVzYdBTqElmeQWtBzhY9aHvX
+r+A02C2j+wKBgHHDo6Gsi57yR5gUPd9vSlCkNtEIrss0DJv5yHMIB+KnaNZcE+NF
+5qFF30Jxyz5RDtxJ9tXcvaeln8lG3XDQKI/MqfDCqTuqo5ImHrfMaW8oA70JxS2p
+gHqGVzkg1aMxsIrmpcdk6olnPExocvWivGdbtzeEjhMALu8Sp6y6nUCFAoGBAK5h
+KLgYw/OMVaQCIMthaa+l6f0s7PMMYe1453H6VBD6qz4/8HPwO7LfG1gzrUYxADgs
+ElVh0UHn/On383nS+i9Ze5Hfyyvwc+LQQURKJPrJQMPJavCptPE7NmiKnYNHK6vr
+yh0l4oxShAklbCJBGvICq4zuVfVfXDeQnDIVTfaPAoGBAMCrZqYdOUhUu+aUqxZq
+qO/TTQxrxftU63jGUg+o042TdgI4KWLn07wvHJ8/E2OqF35eXenvcuKbNLI1l72J
+4cp+3cUv8iAXThTRYEztr5CS/wta4o4CNN8zfjn5dV9AI4Hmt4V7EaGWpBcViGbj
+n0Mhag+dO8DHuenqi1yfMrAt
+-----END PRIVATE KEY-----