Release 3.49.12

Roll up 25 commits since 3.49.11 (July 5): security fixes in the
network-facing parsers (remote stack overflow and uninitialized read in
Content-Type/-Encoding handling, fuzzer-found over-reads in the filter,
URL and IDNA parsers, cache-index bounds, filter-pattern backtracking,
world-readable cookies.txt), crawl-correctness fixes (double-encoded UTF-8
links, gzip-mislabeled bodies, cache reconcile/corruption handling,
orphaned .delayed placeholders), the new --why filter diagnostic, and
build hardening (_FORTIFY_SOURCE, -fstack-protector-strong).

VERSION_INFO 3:3:0 -> 3:4:0: httrackp tail-appends why_url, no layout or
exported-signature break, so soname stays libhttrack.so.3 (revision bump).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsback.c

@@ -68,15 +68,6 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
-/* Which hard quota, if any, is currently aborting the mirror. */
-typedef enum {
-  HTS_MIRROR_LIMIT_NONE = 0,
-  HTS_MIRROR_LIMIT_SIZE,
-  HTS_MIRROR_LIMIT_TIME,
-} hts_mirror_limit;
-
-static hts_mirror_limit back_mirror_limit(httrackp *opt);
-
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -2059,9 +2050,13 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
-        soc = INVALID_SOCKET;             // not opened yet
-        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
+#if HDEBUG
+        printf("back_solve..\n");
+#endif
+        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
+        soc = INVALID_SOCKET;   // pas encore ouverte
+        back_solve(opt, &back[p]);      // préparer
+        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2204,9 +2199,37 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// Resolution is synchronous inside the connect path; no pre-resolve step, so
-// the host is always immediately ready.
-int host_wait(httrackp *opt, lien_back *back) { return 1; }
+// attendre que le host (ou celui du proxy) ait été résolu
+// si c'est un fichier, la résolution est immédiate
+// idem pour ftp://
+void back_solve(httrackp * opt, lien_back * back) {
+  assertf(opt != NULL);
+  assertf(back != NULL);
+  if ((!strfield(back->url_adr, "file://"))
+      && !strfield(back->url_adr, "ftp://")
+    ) {
+    const char *a;
+
+    if (!(back->r.req.proxy.active))
+      a = back->url_adr;
+    else
+      a = back->r.req.proxy.name;
+    assertf(a != NULL);
+    a = jump_protocol_const(a);
+    if (check_hostname_dns(a)) {
+      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
+    } else {
+      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
+    }
+  }
+}
+
+// détermine si le host a pu être résolu
+int host_wait(httrackp * opt, lien_back * back) {
+  // Always synchronous. No more background DNS resolution
+  // (does not really improve performances)
+  return 1;
+}
 #endif
 
 // élimine les fichiers non html en backing (anticipation)
@@ -2393,15 +2416,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
-     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
   if (!back_checkmirror(opt)) {
-    const hts_mirror_limit limit = back_mirror_limit(opt);
-    const char *const reason =
-        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
-    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
-                                    ? "Mirror Size Limit"
-                                    : "Mirror Time Out";
     int aborted = 0;
     unsigned int i;
 
@@ -2415,15 +2432,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
           back_delayed_discard(opt, &back[i]);
         back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, slotmsg);
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
         back[i].status = STATUS_READY;
         back_set_finished(sback, i);
         aborted++;
       }
     }
     if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
-                    reason, aborted);
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
     return;
   }
 
@@ -4128,43 +4145,38 @@ static int back_maxtime_grace(const int maxtime) {
   return maximum(5, minimum(30, maxtime / 10));
 }
 
-/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
-   No floor (unlike the time grace): a size overrun should abort promptly. */
-static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
-
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
-static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
-      HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
-    return HTS_MIRROR_LIMIT_SIZE;
+int back_checkmirror(httrackp * opt) {
+  // Check max size
+  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
+    if (!opt->state.stop) {     /* not yet stopped */
+      hts_log_print(opt, LOG_ERROR,
+                    "More than " LLintP
+                    " bytes have been transferred.. giving up",
+                    (LLint) opt->maxsite);
+      /* cancel mirror smoothly */
+      hts_request_stop(opt, 0);
+    }
+    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
+    /*return 0;
+     */
+  }
+  // Check max time
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
 
-    if (elapsed >= opt->maxtime &&
-        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-      return HTS_MIRROR_LIMIT_TIME;
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
+    }
   }
-  return HTS_MIRROR_LIMIT_NONE;
-}
-
-int back_checkmirror(httrackp *opt) {
-  /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
-      !opt->state.stop) {
-    hts_log_print(opt, LOG_ERROR,
-                  "More than " LLintP
-                  " bytes have been transferred.. giving up",
-                  (LLint) opt->maxsite);
-    hts_request_stop(opt, 0);
-  }
-  if (opt->maxtime > 0 && !opt->state.stop &&
-      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
-    hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                  opt->maxtime);
-    hts_request_stop(opt, 0);
-  }
-  /* hard stop once a cap overruns its grace (callers must stop waiting) */
-  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
+  return 1;                     /* Ok, go on */
 }
 
 // octets transférés + add

src/htsback.h

@@ -138,11 +138,12 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
+void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
-   or -E deadline overruns its grace period (callers must stop waiting). */
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsdns_selftest.c

@@ -297,26 +297,6 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
-  /* "host:port" resolves as the bare host: the cache/connect path strips
-     ":port" before both the backend and the cache key (#181). */
-  mock_reset_calls();
-  {
-    SOCaddr a;
-    char ip[64];
-    const char *err = NULL;
-
-    /* cache miss: the backend is hit with the stripped host, not "host:port" */
-    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* the bare host shares that one cache entry (stripped key) */
-    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* a port variant of an already-cached host also hits, right address */
-    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
-    SOCaddr_inetntoa(ip, sizeof(ip), a);
-    CHECK(strcmp(ip, "1.2.3.4") == 0);
-  }
-
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

tests/42_local-maxsize-slow.test

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# -M byte cap under a slow server (#77): p0 overruns -M at once while p1..p3
-# trickle for a minute; the cap must abort the in-flight transfers, not wait
-# them out. Unfixed, the smooth stop drains the trickle at server pace.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-start=$(date +%s)
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --log-found 'More than 400000 bytes have been transferred.. giving up' \
-    httrack 'BASEURL/bigtrickle/index.html' -M400000 -c4
-wall=$(($(date +%s) - start))
-# trickle runs 60s; a smooth-only stop waits it out, the hard stop lands near 8s.
-# Wall clock is the discriminating check: the log line above fires either way.
-if [ "$wall" -ge 30 ]; then
-    echo "crawl took ${wall}s, -M hard stop did not engage" >&2
-    exit 1
-fi

tests/Makefile.am

@@ -110,7 +110,6 @@ TESTS = \
 	38_local-update-304.test \
 	39_local-delayed-cancel.test \
 	40_local-why.test \
-	41_local-utf8-link.test \
-	42_local-maxsize-slow.test
+	41_local-utf8-link.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -1035,14 +1035,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_bigfile(self):
         self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
 
-    # -M under a slow server (#77): p0 is a fast 640KB file that alone overruns
-    # -M; p1..p3 trickle for a minute. The cap must abort those in-flight
-    # transfers, not wait them out.
-    def route_bigtrickle_index(self):
-        self.send_html(
-            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(4))
-        )
-
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1115,11 +1107,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/bigfiles/p5.bin": route_bigfile,
         "/bigfiles/p6.bin": route_bigfile,
         "/bigfiles/p7.bin": route_bigfile,
-        "/bigtrickle/index.html": route_bigtrickle_index,
-        "/bigtrickle/p0.bin": route_bigfile,
-        "/bigtrickle/p1.bin": route_trickle_page,
-        "/bigtrickle/p2.bin": route_trickle_page,
-        "/bigtrickle/p3.bin": route_trickle_page,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,