Extract the duplicated stats-refresh loop tick into hts_loop_tick()

The wait-for-socket macro body (stats refresh + loop callback) also
existed open-coded at six more sites across htsname.c and htsparse.c,
differing only in the slot index passed to the callback and the abort
action. Collapse all of them onto a shared hts_loop_tick(); each caller
keeps its own abort path.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsback.c

@@ -1359,6 +1359,18 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
 }
 
 // effacer entrée
+/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
+static void back_delayed_discard(httrackp *opt, lien_back *back) {
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  back->r.is_write = 0;
+  if (opt != NULL)
+    url_savename_refname_remove(opt, back->url_adr, back->url_fil);
+  (void) UNLINK(back->url_sav);
+}
+
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p) {
   lien_back *const back = sback->lnk;
@@ -1366,6 +1378,12 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
 
   assertf(p >= 0 && p < back_max);
   if (p >= 0 && p < sback->count) {     // on sait jamais..
+    /* mid-write cancel: drop a .delayed placeholder; real-named partials
+       survive for resume (--continue) */
+    if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
+        (back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
+      back_delayed_discard(opt, &back[p]);
+    }
     // Vérificateur d'intégrité
 #if DEBUG_CHECKINT
     _CHECKINT(&back[p], "Appel back_delete")
@@ -2419,6 +2437,34 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
+  if (!back_checkmirror(opt)) {
+    int aborted = 0;
+    unsigned int i;
+
+    for (i = 0; i < (unsigned int) back_max; i++) {
+      if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
+        if (back[i].r.soc != INVALID_SOCKET) {
+          deletehttp(&back[i].r);
+        }
+        back[i].r.soc = INVALID_SOCKET;
+        /* drop a .delayed placeholder; real partials survive for resume */
+        if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
+          back_delayed_discard(opt, &back[i]);
+        back[i].r.statuscode = STATUSCODE_TIMEOUT;
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        back[i].status = STATUS_READY;
+        back_set_finished(sback, i);
+        aborted++;
+      }
+    }
+    if (aborted > 0)
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
+    return;
+  }
+
   // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
   do_wait = 0;
   gestion_timeout = 0;
@@ -4164,6 +4210,11 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
   return 1;
 }
 
+/* Grace left to the smooth stop before in-flight transfers are aborted. */
+static int back_maxtime_grace(const int maxtime) {
+  return maximum(5, minimum(30, maxtime / 10));
+}
+
 int back_checkmirror(httrackp * opt) {
   // Check max size
   if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
@@ -4180,13 +4231,19 @@ int back_checkmirror(httrackp * opt) {
      */
   }
   // Check max time
-  if ((opt->maxtime > 0)
-      && ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
-    if (!opt->state.stop) {     /* not yet stopped */
-      hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                    opt->maxtime);
-      /* cancel mirror smoothly */
-      hts_request_stop(opt, 0);
+  if (opt->maxtime > 0) {
+    const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
+
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
     }
   }
   return 1;                     /* Ok, go on */

src/htsback.h

@@ -136,6 +136,8 @@ void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htscore.c

@@ -3371,6 +3371,41 @@ int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt) {
   return n;
 }
 
+/* One engine-loop tick: refresh the transfer stats and run the loop callback
+   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
+hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr) {
+  engine_stats();
+  HTS_STAT.stat_nsocket = back_nsoc(sback);
+  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+  return RUN_CALLBACK7(
+             opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
+             (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)
+             ? HTS_TRUE
+             : HTS_FALSE;
+}
+
+/* Single implementation of the historical WAIT_FOR_AVAILABLE_SOCKET macros. */
+hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
+                                      cache_back *cache, int ptr) {
+  const int prev = opt->state._hts_in_html_parsing;
+
+  while (back_pluggable_sockets_strict(sback, opt) <= 0) {
+    opt->state._hts_in_html_parsing = 6;
+    back_wait(sback, opt, cache, 0);
+    /* time limit (-E) exceeded: stop waiting for a socket (#481) */
+    if (!back_checkmirror(opt))
+      break;
+    if (!hts_loop_tick(sback, opt, -1, ptr))
+      return HTS_FALSE;
+  }
+  opt->state._hts_in_html_parsing = prev;
+  return HTS_TRUE;
+}
+
 int back_pluggable_sockets(struct_back * sback, httrackp * opt) {
   int n;
 

src/htscore.h

@@ -432,6 +432,15 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
 
 int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
 
+/* One engine-loop tick: refresh the transfer stats and run the loop callback
+   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
+hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
+
+/* Wait until a test socket can be plugged, pumping transfers, stats and the
+   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
+hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
+                                      cache_back *cache, int ptr);
+
 /* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
    timestamp seed so it is stable within one gap and rerolls per launch. */
 int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);

src/htsname.c

@@ -74,31 +74,6 @@ static const char *hts_tbdev[] = {
   ""
 };
 
-#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state. _hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    { \
-      if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-        return -1; \
-      } \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
-
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
   int i, j;
@@ -652,11 +627,10 @@ int url_savename(lien_adrfilsave *const afs,
             int has_been_moved = 0;
             lien_adrfil current;
 
-            /* Ensure we don't use too many sockets by using a "testing" one
-               If we have only 1 simultaneous connection authorized, wait for pending download
-               Wait for an available slot 
+            /* Wait for an available test slot, honoring the connection limits
              */
-            URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+            if (!hts_wait_available_socket(sback, opt, cache, ptr))
+              return -1;
 
             /* Rock'in */
             current.adr[0] = current.fil[0] = '\0';
@@ -686,24 +660,11 @@ int url_savename(lien_adrfilsave *const afs,
                   if (ptr >= 0) {
                     back_fillmax(sback, opt, cache, ptr, numero_passe);
                   }
-                  // on est obligé d'appeler le shell pour le refresh..
-                  // Transfer rate
-                  engine_stats();
-
-                  // Refresh various stats
-                  HTS_STAT.stat_nsocket = back_nsoc(sback);
-                  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-                  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-                  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-                  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-                  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-                  if (!RUN_CALLBACK7
-                      (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-                       (int) (time_local() - HTS_STAT.stat_timestart),
-                       &HTS_STAT)) {
+                  if (!hts_loop_tick(sback, opt, b, ptr)) {
                     return -1;
-                  } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {        // cancel 2 ou 1 (cancel parsing)
+                  } else if (opt->state._hts_cancel ||
+                             !back_checkmirror(
+                                 opt)) { // cancel level 2 or 1 (cancel parsing)
                     back_delete(opt, cache, sback, b);  // cancel test
                     stop_looping = 1;
                   }
@@ -768,8 +729,9 @@ int url_savename(lien_adrfilsave *const afs,
                                                 "Loop with HEAD request (during prefetch) at %s%s",
                                                 current.adr, current.fil);
                                 }
-                                // Ajouter
-                                URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+                                if (!hts_wait_available_socket(sback, opt,
+                                                               cache, ptr))
+                                  return -1;
                                 if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) {        // OK
                                   hts_log_print(opt, LOG_DEBUG,
                                                 "(during prefetch) %s (%d) to link %s at %s%s",

src/htsparse.c

@@ -3399,20 +3399,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
           back_fillmax(sback, opt, cache, ptr, numero_passe);
 
-          // Transfer rate
-          engine_stats();
-
-          // Refresh various stats
-          HTS_STAT.stat_nsocket = back_nsoc(sback);
-          HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-          HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-          HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-          HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-          HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-          if (!RUN_CALLBACK7
-              (opt, loop, sback->lnk, sback->count, 0, ptr, opt->lien_tot,
-               (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
+          if (!hts_loop_tick(sback, opt, 0, ptr)) {
             hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
             *stre->exit_xh_ = 1;        // exit requested
             XH_uninit;
@@ -3423,7 +3410,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             nofollow = 1;       // moins violent
             opt->state._hts_cancel = 0;
           }
-
         }
         // refresh the backing system each 2 seconds
         if (engine_stats()) {
@@ -3960,22 +3946,8 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
         {
           back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
 
-          // Transfer rate
-          engine_stats();
-
-          // Refresh various stats
-          HTS_STAT.stat_nsocket = back_nsoc(sback);
-          HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-          HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-          HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-          HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-          HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
           b = 0;
-          if (!RUN_CALLBACK7
-              (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-               (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)
-              || !back_checkmirror(opt)) {
+          if (!hts_loop_tick(sback, opt, b, ptr) || !back_checkmirror(opt)) {
             hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
             *stre->exit_xh_ = 1;        // exit requested
             XH_uninit;
@@ -4077,21 +4049,11 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
     while(opt->state._hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) { // on fait la pause..
       opt->state._hts_in_html_parsing = 6;
       back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */
+      if (!back_checkmirror(opt))
+        break;
 
-      // Transfer rate
-      engine_stats();
-
-      // Refresh various stats
-      HTS_STAT.stat_nsocket = back_nsoc(sback);
-      HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-      HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-      if (!RUN_CALLBACK7
-          (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-           (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
+      if (!hts_loop_tick(sback, opt, b, ptr)) {
         hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
         *stre->exit_xh_ = 1;    // exit requested
         XH_uninit;
@@ -4278,26 +4240,12 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
           freet(s);
         }
 
-        // Transfer rate
-        engine_stats();
-
-        // Refresh various stats
-        HTS_STAT.stat_nsocket = back_nsoc(sback);
-        HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-        HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-        HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-        HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-        HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-        if (!RUN_CALLBACK7
-            (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-             (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
+        if (!hts_loop_tick(sback, opt, b, ptr)) {
           hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
           *stre->exit_xh_ = 1;  // exit requested
           XH_uninit;
           return 0;
         }
-
       }
 
 #if HTS_POLL
@@ -4530,10 +4478,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
 
-      /*
-         Wait for an available slot 
-       */
-      WAIT_FOR_AVAILABLE_SOCKET();
+      /* Wait for an available slot */
+      if (!hts_wait_available_socket(sback, opt, cache, ptr))
+        return -1;
 
       /* We can lookup directly in the cache to speedup this mess */
       if (opt->delayed_cached) {
@@ -4679,29 +4626,14 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           if (ptr >= 0) {
             back_fillmax(sback, opt, cache, ptr, numero_passe);
           }
-          // on est obligé d'appeler le shell pour le refresh..
-          {
-
-            // Transfer rate
-            engine_stats();
-
-            // Refresh various stats
-            HTS_STAT.stat_nsocket = back_nsoc(sback);
-            HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-            HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-            HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-            HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-            HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-            if (!RUN_CALLBACK7
-                (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-                 (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
-              back_set_unlocked(sback, b);
-              return -1;
-            } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {      // cancel 2 ou 1 (cancel parsing)
-              back_delete(opt, cache, sback, b);        // cancel test
-              break;
-            }
+          if (!hts_loop_tick(sback, opt, b, ptr)) {
+            back_set_unlocked(sback, b);
+            return -1;
+          } else if (opt->state._hts_cancel ||
+                     !back_checkmirror(
+                         opt)) { // cancel level 2 or 1 (cancel parsing)
+            back_delete(opt, cache, sback, b); // cancel test
+            break;
           }
         } while (
             /* dns/connect/request */

src/htsparse.h

@@ -175,27 +175,4 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
   /* Apply changes */ \
   * str->ptr_ = ptr
 
-#define WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state._hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, -1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-      return -1; \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
-
 #endif

tests/34_local-maxtime.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# -E time limit (#481): server pages trickle for minutes; the engine must stop
+# on its own at -E plus grace, aborting the in-flight transfers.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
+start=$(date +%s)
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --skip-delayed-audit \
+    --log-found 'More than 2 seconds passed' \
+    httrack 'BASEURL/trickle/index.html' -E2 -c4
+wall=$(($(date +%s) - start))
+# hard stop is due at -E2 + 5s grace; near TRICKLE_SECONDS means it never fired
+if [ "$wall" -ge 30 ]; then
+    echo "crawl took ${wall}s, -E hard stop did not engage" >&2
+    exit 1
+fi

tests/Makefile.am

@@ -96,6 +96,7 @@ TESTS = \
 	30_local-fragment-link.test \
 	31_local-javaclass.test \
 	32_local-cdispo.test \
-	33_local-delayed.test
+	33_local-delayed.test \
+	34_local-maxtime.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -92,6 +92,7 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
 # --- parse leading control flags --------------------------------------------
 declare -a audit=()
 declare -a cookies=()
+skip_delayed_audit=""
 scheme=http
 pos=0
 args=("$@")
@@ -116,6 +117,9 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
+    --skip-delayed-audit)
+        skip_delayed_audit=1
+        ;;
     --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -246,12 +250,15 @@ done
 test -n "$hostroot" || die "could not find host root under $out"
 debug "host root: $hostroot"
 
-# A completed crawl must leave no .delayed temporaries (issue #107)
-info "checking for leftover .delayed files"
-leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
-if test -z "$leftovers"; then result "OK"; else
-    result "leftover: $leftovers"
-    exit 1
+# A completed crawl must leave no .delayed temporaries (issue #107).
+# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
+if test -z "$skip_delayed_audit"; then
+    info "checking for leftover .delayed files"
+    leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
+    if test -z "$leftovers"; then result "OK"; else
+        result "leftover: $leftovers"
+        exit 1
+    fi
 fi
 
 # --- audit -------------------------------------------------------------------

tests/local-server.py

@@ -464,6 +464,30 @@ class Handler(SimpleHTTPRequestHandler):
     def route_delayed_empty(self):
         self.send_raw(b"", "text/html")  # 200 + Content-Length: 0
 
+    # -E time-limit (#481): pages that trickle far longer than any -E budget,
+    # so only an engine-side abort can end the crawl.
+    TRICKLE_SECONDS = 60
+
+    def route_trickle_index(self):
+        self.send_html(
+            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
+        )
+
+    def route_trickle_page(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(2 * self.TRICKLE_SECONDS))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            for _ in range(self.TRICKLE_SECONDS):
+                self.wfile.write(b"xy")
+                self.wfile.flush()
+                time.sleep(1.0)
+        except OSError:
+            pass
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -509,6 +533,15 @@ class Handler(SimpleHTTPRequestHandler):
         "/cdispo/fetch.php": route_cdispo,
         "/cdispo/evil.php": route_cdispo,
         "/delayed/index.html": route_delayed_index,
+        "/trickle/index.html": route_trickle_index,
+        "/trickle/p0.bin": route_trickle_page,
+        "/trickle/p1.bin": route_trickle_page,
+        "/trickle/p2.bin": route_trickle_page,
+        "/trickle/p3.bin": route_trickle_page,
+        "/trickle/p4.bin": route_trickle_page,
+        "/trickle/p5.bin": route_trickle_page,
+        "/trickle/p6.bin": route_trickle_page,
+        "/trickle/p7.bin": route_trickle_page,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,