Isolate bigcrawl in a second pass on macOS to parallelize the rest

macOS was the CI bottleneck (~460s serial, the ~8min total): its loopback drops
fetches when bigcrawl's sustained -c8 crawl competes with other crawls, so a
parallel make check flaked bigcrawl's exact file count. Rather than keep the
whole macOS suite serial, run everything except bigcrawl in parallel (2x cores,
capped 16), then bigcrawl alone in a second pass, which is the serial-safe
condition green on master. Linux still runs the full suite in parallel.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,7 +55,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -99,7 +101,8 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          make check
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -133,7 +136,15 @@ jobs:
         run: make -j"$(sysctl -n hw.ncpu)"
 
       - name: Test
-        run: make check
+        # bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
+        # it competes with other crawls, flaking its exact file count (the #527
+        # macOS drop). Run everything else in parallel, then bigcrawl alone (its
+        # serial-safe condition). Linux tolerates the full parallel run.
+        run: |
+          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
+          make check -j"$jobs" TESTS="$rest"
+          make check TESTS=36_local-bigcrawl.test
 
       - name: Print the test log on failure
         if: failure()
@@ -171,7 +182,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -226,7 +239,9 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -272,7 +287,8 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          make check TESTS="$tests"
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs" TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
@@ -347,7 +363,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()

AGENTS.md

@@ -6,8 +6,19 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make && make check`. Or run
-  `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
+  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
+  automake's parallel harness and each crawl test binds its own ephemeral-port
+  server, so `-j` never contends and a multi-minute serial run drops to
+  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
+  automatically; only a test slower than the current longest raises the floor.
+  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
+  spend much of their wall time asleep (server trickles, httrack self-pacing),
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
+  36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
+  the macOS loopback when it competes with other crawls and flakes its exact
+  file count (Linux tolerates the full parallel run).
+  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

src/htsback.c

@@ -68,6 +68,15 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
+/* Which hard quota, if any, is currently aborting the mirror. */
+typedef enum {
+  HTS_MIRROR_LIMIT_NONE = 0,
+  HTS_MIRROR_LIMIT_SIZE,
+  HTS_MIRROR_LIMIT_TIME,
+} hts_mirror_limit;
+
+static hts_mirror_limit back_mirror_limit(httrackp *opt);
+
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -524,22 +533,22 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
+static int create_back_tmpfile(httrackp *opt, lien_back *const back,
+                               const char *ext) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
-             back->url_sav);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
+             back->url_sav, ext);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
+      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
                     back->tmpfile);
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
-             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
-             opt->state.tmpnameid++);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
+             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -547,6 +556,32 @@ static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
   return 0;
 }
 
+/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
+   existing file moved the good copy to back->tmpfile before truncating url_sav.
+   commit keeps the new file and drops the backup; else restore it so an aborted
+   transfer leaves the previous copy intact. Skips the zlib .z temp. */
+static void back_finalize_backup(httrackp *opt, lien_back *const back,
+                                 const hts_boolean commit) {
+  if (back->tmpfile == NULL || back->r.compressed)
+    return;
+  if (commit) {
+    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
+  } else {
+    if (back->r.out != NULL) {
+      fclose(back->r.out);
+      back->r.out = NULL;
+    }
+    (void) UNLINK(back->url_sav); /* drop the failed partial */
+    /* On rename failure keep the backup: it still holds the good copy, so
+       losing it would be worse than an orphaned temp. */
+    if (RENAME(back->tmpfile, back->url_sav) != 0)
+      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                    "could not restore %s; previous copy kept as %s",
+                    back->url_sav, back->tmpfile);
+  }
+  back->tmpfile = NULL;
+}
+
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -582,6 +617,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
+      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -600,7 +636,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p]) == 0) {
+              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -673,6 +709,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
+        /* Body fully received: keep the freshly written url_sav, drop the
+           backup of the previous copy. */
+        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -901,6 +940,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
+  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
+     previous copy from the backup. */
+  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -2050,13 +2092,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-#if HDEBUG
-        printf("back_solve..\n");
-#endif
-        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
-        soc = INVALID_SOCKET;   // pas encore ouverte
-        back_solve(opt, &back[p]);      // préparer
-        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
+        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
+        soc = INVALID_SOCKET;             // not opened yet
+        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2199,37 +2237,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// attendre que le host (ou celui du proxy) ait été résolu
-// si c'est un fichier, la résolution est immédiate
-// idem pour ftp://
-void back_solve(httrackp * opt, lien_back * back) {
-  assertf(opt != NULL);
-  assertf(back != NULL);
-  if ((!strfield(back->url_adr, "file://"))
-      && !strfield(back->url_adr, "ftp://")
-    ) {
-    const char *a;
-
-    if (!(back->r.req.proxy.active))
-      a = back->url_adr;
-    else
-      a = back->r.req.proxy.name;
-    assertf(a != NULL);
-    a = jump_protocol_const(a);
-    if (check_hostname_dns(a)) {
-      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
-    } else {
-      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
-    }
-  }
-}
-
-// détermine si le host a pu être résolu
-int host_wait(httrackp * opt, lien_back * back) {
-  // Always synchronous. No more background DNS resolution
-  // (does not really improve performances)
-  return 1;
-}
+// Resolution is synchronous inside the connect path; no pre-resolve step, so
+// the host is always immediately ready.
+int host_wait(httrackp *opt, lien_back *back) { return 1; }
 #endif
 
 // élimine les fichiers non html en backing (anticipation)
@@ -2416,9 +2426,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
-     starves (#481). FTP slots stay, their thread owns the socket. */
+  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
+     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
   if (!back_checkmirror(opt)) {
+    const hts_mirror_limit limit = back_mirror_limit(opt);
+    const char *const reason =
+        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
+    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
+                                    ? "Mirror Size Limit"
+                                    : "Mirror Time Out";
     int aborted = 0;
     unsigned int i;
 
@@ -2432,15 +2448,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
           back_delayed_discard(opt, &back[i]);
         back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        strcpybuff(back[i].r.msg, slotmsg);
         back[i].status = STATUS_READY;
         back_set_finished(sback, i);
         aborted++;
       }
     }
     if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING,
-                    "time limit reached, %d transfer(s) aborted", aborted);
+      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
+                    reason, aborted);
     return;
   }
 
@@ -2930,7 +2946,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i]) == 0) {
+                              if (create_back_tmpfile(opt, &back[i], "z") ==
+                                  0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -2943,6 +2960,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
+                              /* Re-fetch over an existing file (#77 follow-up):
+                                 move the good copy aside before truncating it
+                                 so an aborted transfer can restore it. url_sav
+                                 is still written normally (file list intact).
+                               */
+                              back[i].tmpfile = NULL;
+                              if (fexist_utf8(back[i].url_sav)) {
+                                if (create_back_tmpfile(opt, &back[i], "bak") !=
+                                        0 ||
+                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
+                                        0) {
+                                  back[i].tmpfile = NULL;
+                                }
+                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3509,22 +3540,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // 'do not erase already downloaded file'
-                  // on an updated file
-                  // with an error : consider a 304 error
-                  if (!opt->delete_old) {
-                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
-                        && !back[i].testmode) {
-                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
-                        hts_log_print(opt, LOG_DEBUG,
-                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
-                                      back[i].r.statuscode, back[i].r.msg,
-                                      back[i].url_adr, back[i].url_fil);
-                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                        deletehttp(&back[i].r);
-                        back[i].r.soc = INVALID_SOCKET;
-                      }
-                    }
+                  // On update, keep the good copy on error (mask as 304); skip
+                  // resume paths so a stale-partial 416 still re-fetches.
+                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
+                      back[i].is_update && !back[i].testmode &&
+                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
+                      fexist_utf8(back[i].url_sav)) {
+                    hts_log_print(opt, LOG_DEBUG,
+                                  "Error %d (%s) ignored on update, keeping "
+                                  "existing copy: %s%s",
+                                  back[i].r.statuscode, back[i].r.msg,
+                                  back[i].url_adr, back[i].url_fil);
+                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                    deletehttp(&back[i].r);
+                    back[i].r.soc = INVALID_SOCKET;
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
@@ -4145,38 +4174,43 @@ static int back_maxtime_grace(const int maxtime) {
   return maximum(5, minimum(30, maxtime / 10));
 }
 
-int back_checkmirror(httrackp * opt) {
-  // Check max size
-  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
-    if (!opt->state.stop) {     /* not yet stopped */
-      hts_log_print(opt, LOG_ERROR,
-                    "More than " LLintP
-                    " bytes have been transferred.. giving up",
-                    (LLint) opt->maxsite);
-      /* cancel mirror smoothly */
-      hts_request_stop(opt, 0);
-    }
-    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
-    /*return 0;
-     */
-  }
-  // Check max time
+/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
+   No floor (unlike the time grace): a size overrun should abort promptly. */
+static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
+
+/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
+static hts_mirror_limit back_mirror_limit(httrackp *opt) {
+  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
+      HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
+    return HTS_MIRROR_LIMIT_SIZE;
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
 
-    if (elapsed >= opt->maxtime) {
-      if (!opt->state.stop) { /* not yet stopped */
-        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                      opt->maxtime);
-        /* cancel mirror smoothly */
-        hts_request_stop(opt, 0);
-      }
-      /* smooth stop starved past the grace period: stop waiting (#481) */
-      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-        return 0;
-    }
+    if (elapsed >= opt->maxtime &&
+        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+      return HTS_MIRROR_LIMIT_TIME;
   }
-  return 1;                     /* Ok, go on */
+  return HTS_MIRROR_LIMIT_NONE;
+}
+
+int back_checkmirror(httrackp *opt) {
+  /* request a smooth stop the first time each cap is reached */
+  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
+      !opt->state.stop) {
+    hts_log_print(opt, LOG_ERROR,
+                  "More than " LLintP
+                  " bytes have been transferred.. giving up",
+                  (LLint) opt->maxsite);
+    hts_request_stop(opt, 0);
+  }
+  if (opt->maxtime > 0 && !opt->state.stop &&
+      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
+    hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                  opt->maxtime);
+    hts_request_stop(opt, 0);
+  }
+  /* hard stop once a cap overruns its grace (callers must stop waiting) */
+  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
 }
 
 // octets transférés + add

src/htsback.h

@@ -138,12 +138,11 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
-void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
-   the -E deadline overran its grace period (callers must stop waiting). */
+/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
+   or -E deadline overruns its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htscore.c

@@ -1904,8 +1904,11 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
-          if (r.adr != NULL || r.size == 0) {
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
+          // An empty body must not overwrite the file when the transfer failed
+          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
+          // copy to 0 (#77 follow-up).
+          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1926,7 +1929,6 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
-
         }
 
         /* Parsing of other media types (java, ram..) */

src/htsdns_selftest.c

@@ -297,6 +297,26 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
+  /* "host:port" resolves as the bare host: the cache/connect path strips
+     ":port" before both the backend and the cache key (#181). */
+  mock_reset_calls();
+  {
+    SOCaddr a;
+    char ip[64];
+    const char *err = NULL;
+
+    /* cache miss: the backend is hit with the stripped host, not "host:port" */
+    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* the bare host shares that one cache entry (stripped key) */
+    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* a port variant of an already-cached host also hits, right address */
+    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
+    SOCaddr_inetntoa(ip, sizeof(ip), a);
+    CHECK(strcmp(ip, "1.2.3.4") == 0);
+  }
+
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsglobal.h

@@ -56,6 +56,10 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
+// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
+#include <stdint.h>
+#include <inttypes.h>
+
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -111,29 +115,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef SIZEOF_LONG_LONG
-#if SIZEOF_LONG_LONG == 8
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-#ifndef HTS_LONGLONG
-#ifdef __sun
-#define HTS_LONGLONG 0
-#endif
-#ifdef __osf__
-#define HTS_LONGLONG 0
-#endif
-#ifdef __linux
-#define HTS_LONGLONG 1
-#endif
-#ifdef _WIN32
-#define HTS_LONGLONG 1
-#endif
-#endif
-#endif
-
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -326,59 +307,11 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef HTS_NO_64_BIT
-#define HTS_LONGLONG 0
-#else
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-/* Wide integer types, chosen per platform.
-   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
-           plain int where 64-bit is unavailable).
-   TStamp: timestamp/duration in the same width (a double in the no-64-bit
-           fallback).
-   LLintP: the printf conversion for an LLint. */
-#if HTS_LONGLONG
-#ifdef LLINT_FORMAT
-typedef LLINT_TYPE LLint;
-
-typedef LLINT_TYPE TStamp;
-
-#define LLintP LLINT_FORMAT
-#else
-
-#ifdef _WIN32
-typedef __int64 LLint;
-
-typedef __int64 TStamp;
-
-#define LLintP "%I64d"
-#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
-       defined(__64BIT__))
-
-typedef long int LLint;
-
-typedef long int TStamp;
-
-#define LLintP "%ld"
-#else
-typedef long long int LLint;
-
-typedef long long int TStamp;
-
-#define LLintP "%lld"
-#endif
-
-#endif /* HTS_LONGLONG */
-
-#else
-typedef int LLint;
-
-#define LLintP "%d"
-typedef double TStamp;
-#endif
+/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
+typedef int64_t LLint;
+typedef int64_t TStamp;
+/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
+#define LLintP "%" PRId64
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain

src/htslib.c

@@ -3038,12 +3038,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
-#ifdef HTS_LONGLONG
-#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#endif
+#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
+#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintPiB                                                             \
+  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3052,9 +3051,7 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  }
-#ifdef HTS_LONGLONG
-  else if (n < ToLLintGiB) {
+  } else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3071,13 +3068,6 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
-#else
-  else {
-    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
-            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
-    strcpybuff(strc->buff2, "MiB");
-  }
-#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;

tests/36_local-bigcrawl.test

@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors 4 --files 361 \
+    --errors-content 4 --files 361 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \

tests/42_local-maxsize-slow.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# -M byte cap under a slow server (#77): p0 overruns -M at once while p1..p3
+# trickle for a minute; the cap must abort the in-flight transfers, not wait
+# them out. Unfixed, the smooth stop drains the trickle at server pace.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+start=$(date +%s)
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    httrack 'BASEURL/bigtrickle/index.html' -M400000 -c4
+wall=$(($(date +%s) - start))
+# trickle runs 60s; a smooth-only stop waits it out, the hard stop lands near 8s.
+# Wall clock is the discriminating check: the log line above fires either way.
+if [ "$wall" -ge 30 ]; then
+    echo "crawl took ${wall}s, -M hard stop did not engage" >&2
+    exit 1
+fi

tests/43_local-update-truncate.test

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# -M hard-abort must not destroy already-complete files (#77 follow-up).
+# Pass 1 mirrors bigtrunc fully. Pass 2 re-fetches under --update -M: fast.bin
+# overruns the cap, its abort tears down slow.bin's stalled re-fetch. Both were
+# complete on disk from pass 1, and both must survive the aborted update:
+#   - slow.bin's re-fetch is incomplete; the direct-to-disk write must not be
+#     left truncated (the previous copy is restored).
+#   - fast.bin fully arrives but is torn down; the aborted slot must not overwrite
+#     it with an empty body.
+# Unfixed, slow.bin is truncated and fast.bin is zeroed while the cache still
+# records them complete.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update -M400000' \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --found bigtrunc/slow.bin \
+    --file-min-bytes bigtrunc/slow.bin 655360 \
+    --file-min-bytes bigtrunc/fast.bin 655360 \
+    httrack 'BASEURL/bigtrunc/index.html' -c2

tests/44_local-update-errormask.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
+# keep its good local copy, not be overwritten by the error body nor purged.
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --rerun \
+    --found 'errmask/keep.dat' \
+    --file-min-bytes 'errmask/keep.dat' 1024 \
+    --file-matches 'errmask/keep.dat' '^KEEP' \
+    --file-not-matches 'errmask/keep.dat' 'error 403' \
+    httrack 'BASEURL/errmask/index.html'

tests/Makefile.am

@@ -110,6 +110,9 @@ TESTS = \
 	38_local-update-304.test \
 	39_local-delayed-cancel.test \
 	40_local-why.test \
-	41_local-utf8-link.test
+	41_local-utf8-link.test \
+	42_local-maxsize-slow.test \
+	43_local-update-truncate.test \
+	44_local-update-errormask.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -13,15 +13,21 @@
 #
 # Usage:
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
-#       --errors N --files N --found PATH ... --directory PATH ... \
+#       [--rerun-args 'ARGS'] \
+#       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
-#       --max-mirror-bytes N \
+#       --file-min-bytes PATH N --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
+# --errors counts every "Error:" log line; --errors-content drops transient
+# network failures (codes -2..-6) that flake on busy loopback under -c8.
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
+# --file-min-bytes asserts a mirrored file (PATH) is at least N bytes.
+# --rerun-args runs a second pass (same server and mirror dir) with the given
+# extra httrack args appended, e.g. an --update run under a cap.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
@@ -39,6 +45,7 @@ key="${testdir}/server.key"
 tls=
 verbose=
 rerun=
+rerun_args=
 rerun_dead=
 tmpdir=
 serverpid=
@@ -122,7 +129,11 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
-    --errors | --files)
+    --rerun-args)
+        pos=$((pos + 1))
+        rerun_args="${args[$pos]}"
+        ;;
+    --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -130,7 +141,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --file-matches | --file-not-matches)
+    --file-matches | --file-not-matches | --file-min-bytes)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
         pos=$((pos + 2))
         ;;
@@ -241,6 +252,25 @@ if test -n "$rerun"; then
     fi
 fi
 
+# --- optional second pass with extra args (e.g. an --update run under a cap) ---
+# Same server and mirror dir as the first pass, so the second pass sees the
+# cache the first pass wrote. Used to exercise re-fetch/update behaviour.
+if test -n "$rerun_args"; then
+    read -ra extra <<<"$rerun_args"
+    info "re-running httrack with ${rerun_args}"
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
+        "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
+    crawlpid=$!
+    wait "$crawlpid"
+    crawlres=$?
+    crawlpid=
+    test "$crawlres" -eq 0 || ! result "second pass exited $crawlres" || {
+        cat "${log}.2" >&2
+        exit 1
+    }
+    result "OK (second pass)"
+fi
+
 # --- optional dead pass: server stopped, the cache must survive the rollback --
 if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
@@ -306,6 +336,14 @@ while test "$i" -lt "${#audit[@]}"; do
         assert_equals "checking errors" "${audit[$i]}" \
             "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
         ;;
+    --errors-content)
+        i=$((i + 1))
+        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
+        # transient network failures (statuscode -2..-6) flake on busy loopback;
+        # the code parens are followed by " at link" or " after N retries at link"
+        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
+        assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
+        ;;
     --files)
         i=$((i + 1))
         nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
@@ -388,6 +426,16 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
+    --file-min-bytes)
+        path="${audit[$((i + 1))]}"
+        i=$((i + 2))
+        sz=$(wc -c <"${hostroot}/${path}" 2>/dev/null | tr -d '[:space:]')
+        info "checking ${path} size ${sz:-0} >= ${audit[$i]} bytes"
+        if test -n "$sz" && test "$sz" -ge "${audit[$i]}"; then result "OK"; else
+            result "file too small (or missing)"
+            exit 1
+        fi
+        ;;
     esac
     i=$((i + 1))
 done

tests/local-server.py

@@ -934,6 +934,33 @@ class Handler(SimpleHTTPRequestHandler):
     def route_mini304_page(self):
         self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
 
+    # --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
+    # on the update fetch must keep its good copy, not be overwritten nor purged.
+    ERRMASK_GOOD = b"KEEP" + b"." * 1020  # 1024 B distinctive non-HTML body
+    ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
+
+    def route_errmask_index(self):
+        self.send_html('\t<a href="keep.dat">keep</a>\n')
+
+    def route_errmask_keep(self):
+        # First crawl (no validator) gets the 1024 B body + Last-Modified; the
+        # update sends a conditional and gets a 403 error page.
+        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
+            self.send_response(403, "Forbidden")
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.ERRMASK_ERR)
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Last-Modified", BIG_LASTMOD)
+        self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.ERRMASK_GOOD)
+
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -1035,6 +1062,48 @@ class Handler(SimpleHTTPRequestHandler):
     def route_bigfile(self):
         self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
 
+    # -M under a slow server (#77): p0 is a fast 640KB file that alone overruns
+    # -M; p1..p3 trickle for a minute. The cap must abort those in-flight
+    # transfers, not wait them out.
+    def route_bigtrickle_index(self):
+        self.send_html(
+            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(4))
+        )
+
+    # -M hard-abort must not destroy an already-complete file (#77 follow-up).
+    # "fast.bin" alone overruns -M and completes; "slow.bin" transfers fully on
+    # its first fetch (initial mirror) but stalls on every later fetch (the
+    # --update re-fetch), so the -M abort tears it down mid-body. An engine that
+    # truncates the good local copy on the aborted re-fetch loses data.
+    slow_seen = 0
+
+    def route_bigtrunc_index(self):
+        self.send_html('\t<a href="fast.bin">fast</a>\n\t<a href="slow.bin">slow</a>\n')
+
+    def route_bigtrunc_slow(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(self.BIGFILE_BYTES))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        # Count body fetches only, so a stray HEAD can't shift which pass stalls.
+        Handler.slow_seen += 1
+        first = Handler.slow_seen == 1
+        try:
+            if first:
+                self.wfile.write(b"x" * self.BIGFILE_BYTES)
+                self.wfile.flush()
+            else:
+                self.wfile.write(b"x" * 4096)
+                self.wfile.flush()
+                for _ in range(120):
+                    self.wfile.write(b"x")
+                    self.wfile.flush()
+                    time.sleep(1.0)
+        except OSError:
+            pass
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1107,6 +1176,14 @@ class Handler(SimpleHTTPRequestHandler):
         "/bigfiles/p5.bin": route_bigfile,
         "/bigfiles/p6.bin": route_bigfile,
         "/bigfiles/p7.bin": route_bigfile,
+        "/bigtrickle/index.html": route_bigtrickle_index,
+        "/bigtrickle/p0.bin": route_bigfile,
+        "/bigtrickle/p1.bin": route_trickle_page,
+        "/bigtrickle/p2.bin": route_trickle_page,
+        "/bigtrickle/p3.bin": route_trickle_page,
+        "/bigtrunc/index.html": route_bigtrunc_index,
+        "/bigtrunc/fast.bin": route_bigfile,
+        "/bigtrunc/slow.bin": route_bigtrunc_slow,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,
@@ -1127,6 +1204,8 @@ class Handler(SimpleHTTPRequestHandler):
         "/redir/target.html": route_redir_target,
         "/mini304/index.html": route_mini304_index,
         "/mini304/page.html": route_mini304_page,
+        "/errmask/index.html": route_errmask_index,
+        "/errmask/keep.dat": route_errmask_keep,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------