ParseAll URL detection dies after the first script and misses mid-tag attributes (#497)

* Aggressive parser dies for the rest of the page after the first script

The script automaton state (inscript_state_pos) is reset when a script
or style element is entered, but never when it exits. The dirty-parser
character tracker (parseall_lastc) only advances while that state is
INSCRIPT_START, without checking inscript, so whatever state the
automaton holds at </script> freezes for all following HTML. The '/' of
</script> itself is fed to the automaton before the exit branch runs,
so even a clean script deterministically parks it in INSCRIPT_SLASH:
URL detection in unknown attributes (data-*, content) goes dead after
the first script on the page (#201, #203). A </script> inside a JS
string or comment freezes a quote state with parseall_lastc stuck on
'=', turning stray quoted tokens into phantom fetches instead.

Reset the state at the three live exit sites, mirroring the entry
resets. Regression test proven to fail on the unfixed parser.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

* Quoted attribute values mid-tag were invisible to the aggressive parser

Even with the automaton unfrozen, a quoted string only qualified when
its next non-space character was one of ),;>/+ or end-of-line: in HTML
terms, only the last attribute of a tag (or line) could be detected.
The mid-tag attributes of #201 (data-gifsrc="x.gif" data-poster=...)
were structurally unreachable, making detection depend on attribute
order and source formatting.

In a tag outside any script, a quoted value ends at its closing quote,
so waive the follower requirement there. The arm resolves the owning
attribute name itself and declines for no-detect/xmlns names and
non-attribute quotes: the intag_startattr lookup is unreliable mid-tag
(repointed at every in-tag whitespace, stale on glued attributes), and
adversarial review showed a plain intag arm fetching alt/xmlns values
that master suppressed. Script and event-handler contexts keep the
strict gate.

Tests pin each automaton-exit reset (handler fixtures freeze lastc on
';' so they stay live), the nodetect/xmlns bypass shapes, the frozen-
quote phantom-fetch mode, and the '='-resolution decoys. Runtime
audits in httrack-works/issue-201-203/.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

---------

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/htsback.c

@@ -1362,9 +1362,39 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
   return 0;
 }
 
+/* Move a still-writing .delayed placeholder to its final name (#483). */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname) {
+  hts_boolean renamed;
+
+  if (!back->r.is_write || back->tmpfile != NULL ||
+      !IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
+    return HTS_TRUE; /* nothing bound to the placeholder name */
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
+  if (renamed && (back->status == STATUS_READY ||
+                  (back->r.out = FOPEN(newname, "ab")) != NULL)) {
+    filenote(&opt->state.strc, newname, NULL);
+    hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
+                  newname);
+    return HTS_TRUE;
+  }
+  /* partial lost: drop only what we own (Windows rename won't overwrite) */
+  hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
+                back->url_sav, newname);
+  back->r.statuscode = STATUSCODE_INVALID;
+  strcpybuff(back->r.msg, "Write error on disk");
+  back->r.is_write = 0;
+  (void) UNLINK(renamed ? newname : back->url_sav);
+  return HTS_FALSE;
+}
+
 // effacer entrée
 /* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
-static void back_delayed_discard(httrackp *opt, lien_back *back) {
+void back_delayed_discard(httrackp *opt, lien_back *back) {
   if (back->r.out != NULL) {
     fclose(back->r.out);
     back->r.out = NULL;

src/htsback.h

@@ -113,6 +113,12 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
+/* Discard back's on-disk .delayed placeholder and its refname. */
+void back_delayed_discard(httrackp *opt, lien_back *back);
+/* Move back's .delayed placeholder (and open stream) to newname;
+   HTS_FALSE = file lost, slot flagged in error. */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,

src/htscache.c

@@ -799,11 +799,11 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
           }
 
-          /* A tampered X-Size must be rejected before the size-driven malloc.
-             The alloc casts to int (malloct((int) r.size + 1)), so bound it to
-             [0, INT_MAX): a negative value, or a positive one whose (int) cast
-             truncates negative, would otherwise wrap to a huge allocation. */
-          if (r.size < 0 || r.size >= INT_MAX) {
+          /* A negative X-Size is corrupt; so is one >= INT_MAX when the data
+             is in the zip (the write path asserts int-sized). Headers-only
+             entries legitimately exceed INT_MAX (>2GB body on disk): keep
+             them, or every update would re-fetch the file. */
+          if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
             r.statuscode = STATUSCODE_INVALID;
             strcpybuff(r.msg, "Cache Read Error : Bad Size");
           }
@@ -999,7 +999,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg,
                                "Previous cache file not found (empty filename)");
                   }
-                } else {        /* Read in memory from disk */
+                } else if (r.size >= INT_MAX) { /* too big to read in memory */
+                  r.statuscode = STATUSCODE_INVALID;
+                  strcpybuff(r.msg, "Cache Read Error : Bad Size");
+                } else { /* Read in memory from disk */
                   FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
 
                   if (fp != NULL) {

src/htscache_selftest.c

@@ -1158,6 +1158,46 @@ static void corrupt_build_etag(httrackp *opt) {
   selftest_close(&cache);
 }
 
+/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
+   body on disk): the shape every non-html file is stored with. */
+static void corrupt_build_disk(httrackp *opt) {
+  cache_back cache;
+  htsblk w;
+  char locw[4];
+  char BIGSTK save[HTS_URLMAXSIZE * 2];
+  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
+  char *path;
+  FILE *fp;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+          CORRUPT_ADR "/victim.bin");
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  hts_init_htsblk(&w);
+  w.statuscode = 200;
+  w.size = (LLint) sizeof(corrupt_body_b) - 1;
+  strcpybuff(w.msg, "OK");
+  strcpybuff(w.contenttype, "application/octet-stream");
+  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
+  locw[0] = '\0';
+  w.location = locw;
+  w.is_write = 0;
+  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
+            0 /* all_in_cache */, NULL);
+  selftest_close(&cache);
+  /* the reader only checks this file exists; it never reads it here */
+  path = fconv(catbuff, sizeof(catbuff), save);
+  (void) structcheck(path);
+  fp = FOPEN(path, "wb");
+  assertf(fp != NULL);
+  fclose(fp);
+}
+
 /* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
 static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
                           const char *rep, size_t nth, size_t total) {
@@ -1217,8 +1257,8 @@ static void corrupt_victim_body(httrackp *opt) {
    /canary.html: the victim must be rejected (wantmsg pins which path) and the
    canary must still decode byte-exact, proving one bad entry never taints a
    sibling read. */
-static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
-                                 const char *what) {
+static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
+                                     const char *wantmsg, const char *what) {
   cache_back cache;
   htsblk v, c;
   char BIGSTK lv[HTS_URLMAXSIZE * 2];
@@ -1227,7 +1267,7 @@ static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
 
   selftest_open_for_read(&cache, opt);
   lv[0] = lc[0] = '\0';
-  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.html", "", lv, NULL, 1);
+  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
   if (v.statuscode != STATUSCODE_INVALID) {
     fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
             selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
@@ -1254,6 +1294,34 @@ static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
   return fail;
 }
 
+static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
+                                 const char *what) {
+  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
+}
+
+/* Headers-only probe of the disk victim: must parse OK with the size kept. */
+static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
+                                      const char *what) {
+  cache_back cache;
+  htsblk v;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
+  if (v.statuscode != 200 || v.size != wantsize) {
+    fprintf(stderr,
+            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
+            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
 /* One zip corruption case: build, patch, then check victim+canary in-session.
  */
 static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
@@ -1302,5 +1370,33 @@ int cache_corruption_selftest(httrackp *opt, const char *dir) {
   failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
                                     "X-Size above INT_MAX");
 
+  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
+     is how every >2GB non-html file is stored. It must survive a header probe
+     (or every update re-fetches the file); an in-memory read still rejects. */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
+                                         "headers-only X-Size above INT_MAX");
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size above INT_MAX");
+
+  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483647AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size at INT_MAX");
+
+  /* the negative check must stay global, headers-only included */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: -2147483648AAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "headers-only negative X-Size");
+
   return failures;
 }

src/htsparse.c

@@ -192,6 +192,48 @@ static void hts_automate_lookup(const script_automate *aut) {
   }
 }
 
+/* Attribute name owning the quoted value at 'quote' inside a tag, spanning
+   [name, *nend); NULL when the quote is not an attribute value. */
+static const char *dirty_attr_name(const char *quote, const char *tag_start,
+                                   const char **nend) {
+  const char *a = quote - 1;
+  while (a > tag_start && is_taborspace(*a))
+    a--;
+  if (a == tag_start || *a != '=')
+    return NULL;
+  a--;
+  while (a > tag_start && is_taborspace(*a))
+    a--;
+  *nend = a + 1;
+  while (a > tag_start && *a != '=' && *a != '\"' && *a != '\'' &&
+         !is_realspace(*a))
+    a--;
+  a++;
+  // a name starting right after '<' is the tag name, not an attribute
+  return a < *nend && a > tag_start + 1 ? a : NULL;
+}
+
+/* Accept the in-tag quoted value at 'quote' for dirty parsing? Resolves the
+   owning attribute itself (intag_startattr is unreliable mid-tag) and rejects
+   no-detect/xmlns names. */
+static hts_boolean dirty_attr_detectable(const char *quote,
+                                         const char *tag_start) {
+  const char *nend;
+  const char *name = dirty_attr_name(quote, tag_start, &nend);
+  int i;
+  if (name == NULL)
+    return HTS_FALSE;
+  for (i = 0; strnotempty(hts_nodetect[i]); i++) {
+    const int l = strfield(name, hts_nodetect[i]);
+    if (l && name + l == nend)
+      return HTS_FALSE;
+  }
+  i = strfield(name, "xmlns");
+  if (i && (name + i == nend || name[i] == ':'))
+    return HTS_FALSE;
+  return HTS_TRUE;
+}
+
 /* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
 static void hts_automate_increment(const script_automate *aut, int steps) {
   while (steps > 0) {
@@ -754,6 +796,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           ) {
           if (inscript_tag) {
             inscript_tag = inscript = 0;
+            // reset the automaton on exit or its state leaks into plain HTML
+            inscript_state_pos = INSCRIPT_START;
             intag = 0;
             incomment = 0;
             intag_start_valid = 0;
@@ -837,6 +881,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 if (*html == inscript_tag_lastc) {
                   /* sortir */
                   inscript_tag = inscript = 0;
+                  inscript_state_pos = INSCRIPT_START;
                   incomment = 0;
                   if (opt->parsedebug) {
                     HT_ADD("<@@ /inscript @@>");
@@ -1293,6 +1338,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     a--;
                     if (*a == '<') {    // sûr que c'est un tag?
                       inscript = 0;
+                      inscript_state_pos = INSCRIPT_START;
                       if (opt->parsedebug) {
                         HT_ADD("<@@ /inscript @@>");
                       }
@@ -1568,8 +1614,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           while(is_taborspace(*a))
                             a++;
                           c = *a;
-                          if (strchr("),;>/+\r\n", c)) {        // exemple: ..img.gif";
-                            // le / est pour funct("img.gif" /* URL */);
+                          // in-tag, an attribute value ends at its quote: no
+                          // delimiter required after it (mid-tag attrs, #201)
+                          if (strchr("),;>/+\r\n", c) ||
+                              (intag && !inscript && intag_start_valid &&
+                               dirty_attr_detectable(html, intag_start))) {
+                            // '/' covers a value followed by a JS comment
                             char BIGSTK tempo[HTS_URLMAXSIZE * 2];
                             char type[256];
                             int url_ok = 0;     // url valide?
@@ -4566,6 +4616,12 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           back_maydelete(opt, cache, sback, b);    // cancel
           b = -1;
 
+          /* the cancel may leave the now-unreferenced placeholder on disk
+           * (#483) */
+          if (fexist_utf8(delayed_back.url_sav)) {
+            back_delayed_discard(opt, &delayed_back);
+          }
+
           /* Recompute filename with MIME type */
           afs->save[0] = '\0';
           url_savename(afs, former, heap(ptr)->adr,
@@ -4782,6 +4838,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
 
           /* Still have a back reference */
           if (b >= 0) {
+            /* move a still-writing placeholder before the url_sav patch
+               blinds every cleanup to it (#483) */
+            back_delayed_rename(opt, &back[b], afs->save);
             /* patch url_sav BEFORE finalize: it records/caches under this name
              */
             strcpybuff(back[b].url_sav, afs->save);

tests/01_engine-parse.test

@@ -106,8 +106,6 @@ grep -Eq 'srcset="j\.gif 2x"' "$saved" ||
 # inline style attribute, with the URL unquoted, double-quoted and single-quoted
 # (the quote style is preserved on rewrite). No-detect attributes (title, alt,
 # ...) are left untouched. Asserted by rewrite (deterministic), not download.
-# data-* (#201/#203) is omitted: its detection is currently nondeterministic and
-# can't be locked yet.
 site2="$tmp/attrs"
 mkdir -p "$site2"
 for f in xl ibg ibgs cex cexd cexs tt; do gif "$site2/$f.gif"; done
@@ -352,4 +350,64 @@ found "v.webm" "$out10"
 found "subs.vtt" "$out10"
 notfound "plain.gif" "$out10"
 
+# Unknown-attr (data-*) URLs (#201/#203): the script automaton state must reset at
+# </script>, or detection dies for the rest of the page after the first script.
+site11="$tmp/dataattr"
+mkdir -p "$site11"
+for f in pre post mid spdata handler handler2 jsdecoy jsdecoy2 nodecoy \
+    spalt glalt spxml jtail1 jtail2 textdecoy tagdecoy phantom; do gif "$site11/$f.gif"; done
+cat >"$site11/index.html" <<EOF
+<html><body>
+<img data-pre="file://$site11/pre.gif">
+<script>var x = 1;</script>
+<img data-post="file://$site11/post.gif">
+<img data-mid="file://$site11/mid.gif" alt="mid-tag attr, no delimiter after">
+<img data-sp = "file://$site11/spdata.gif" q=1>
+<img alt="nodecoy.gif" src="pre.gif">
+<img alt = "spalt.gif" id=x>
+<img src="pre.gif"alt="glalt.gif" id=x>
+<p xmlns:bar = "spxml.gif" id=x></p>
+<div data-json='["jtail1.gif","jtail2.gif"]' q=1></div>
+<p>t = "textdecoy.gif" q</p>
+<img ="tagdecoy.gif" q>
+<a onclick='q = 1; "h1'>x</a>
+<img data-h1="file://$site11/handler.gif">
+<a onclick='w = 1; "h2>x</a>
+<img data-h2="file://$site11/handler2.gif">
+<script>var s; s = 1; "jsdecoy.gif";</script>
+<script>var s2 = "jsdecoy2.gif" x;</script>
+<script>var f = "</script>
+<img alt="y" "phantom.gif">
+</body></html>
+EOF
+out11="$tmp/dataattr-out"
+crawl "$site11/index.html" "$out11"
+saved11=$(savedhtml "$out11")
+test -n "$saved11" || ! echo "FAIL: saved dataattr page not found" || exit 1
+grep -Fq 'data-pre="pre.gif"' "$saved11" ||
+    ! echo "FAIL #201: data-* URL before any script not detected/rewritten" || exit 1
+grep -Fq 'data-post="post.gif"' "$saved11" ||
+    ! echo "FAIL #201: data-* URL after a script not detected (state leak)" || exit 1
+grep -Fq 'data-mid="mid.gif"' "$saved11" ||
+    ! echo "FAIL #201: mid-tag data-* URL not detected" || exit 1
+grep -Fq 'data-sp = "spdata.gif"' "$saved11" ||
+    ! echo "FAIL #201: spaced-= data-* URL not detected" || exit 1
+found "handler.gif" "$out11"  # automaton reset at the handler-terminator exit
+found "handler2.gif" "$out11" # ... and at the '>' exit of an unterminated handler
+# a JS string not preceded by =/(/, is still ignored after the reset
+notfound "jsdecoy.gif" "$out11"
+# in-script, the strict follower gate still applies (intag is 1 in script bodies)
+notfound "jsdecoy2.gif" "$out11"
+# no-detect attrs stay exempt, incl. spaced-'=' and glued-attr forms
+notfound "nodecoy.gif" "$out11"
+notfound "spalt.gif" "$out11"
+notfound "glalt.gif" "$out11"
+notfound "spxml.gif" "$out11"
+# not '='-preceded: comma-list tails and out-of-tag text tokens stay ignored
+notfound "jtail2.gif" "$out11"
+notfound "textdecoy.gif" "$out11"
+notfound "tagdecoy.gif" "$out11" # a '=' glued to the tag name is not an attr
+# a </script> inside a JS string must not cause phantom fetches later
+notfound "phantom.gif" "$out11"
+
 exit 0

tests/34_local-maxtime.test

@@ -7,10 +7,8 @@ set -euo pipefail
 
 : "${top_srcdir:=..}"
 
-# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
 start=$(date +%s)
 bash "$top_srcdir/tests/local-crawl.sh" \
-    --skip-delayed-audit \
     --log-found 'More than 2 seconds passed' \
     httrack 'BASEURL/trickle/index.html' -E2 -c4
 wall=$(($(date +%s) - start))

tests/39_local-delayed-cancel.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Cancelled delayed-type-checks must not orphan .delayed placeholders (#483).
+# Timing-dependent (hence two tries); -A keeps the window reachable.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+for _ in 1 2; do
+    bash "$top_srcdir/tests/local-crawl.sh" \
+        httrack 'BASEURL/dcancel/index.html' -E1 -c4 -A25000
+done

tests/Makefile.am

@@ -103,6 +103,7 @@ TESTS = \
 	35_local-maxsize.test \
 	36_local-bigcrawl.test \
 	37_local-cache-outage.test \
-	38_local-update-304.test
+	38_local-update-304.test \
+	39_local-delayed-cancel.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -97,7 +97,6 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
 # --- parse leading control flags --------------------------------------------
 declare -a audit=()
 declare -a cookies=()
-skip_delayed_audit=""
 scheme=http
 pos=0
 args=("$@")
@@ -123,9 +122,6 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
-    --skip-delayed-audit)
-        skip_delayed_audit=1
-        ;;
     --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -293,15 +289,12 @@ done
 test -n "$hostroot" || die "could not find host root under $out"
 debug "host root: $hostroot"
 
-# A completed crawl must leave no .delayed temporaries (issue #107).
-# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
-if test -z "$skip_delayed_audit"; then
-    info "checking for leftover .delayed files"
-    leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
-    if test -z "$leftovers"; then result "OK"; else
-        result "leftover: $leftovers"
-        exit 1
-    fi
+# No crawl, even a cancelled one, may leave .delayed temporaries (#107, #483).
+info "checking for leftover .delayed files"
+leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
+if test -z "$leftovers"; then result "OK"; else
+    result "leftover: $leftovers"
+    exit 1
 fi
 
 # --- audit -------------------------------------------------------------------

tests/local-server.py

@@ -913,6 +913,26 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
+    # #483: trickled .bin pages so the -E stop lands in the type waiter's
+    # unlock-to-patch window with body bytes pending.
+    def route_dcancel_index(self):
+        self.send_bin_index()
+
+    def route_dcancel_page(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", "4096")
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            for _ in range(32):
+                self.wfile.write(b"z" * 128)
+                self.wfile.flush()
+                time.sleep(0.05)
+        except OSError:
+            pass
+
     # -M byte cap (#77): large fast files so a crawl overruns -M immediately.
     BIGFILE_BYTES = 640 * 1024
 
@@ -976,6 +996,15 @@ class Handler(SimpleHTTPRequestHandler):
         "/trickle/p5.bin": route_trickle_page,
         "/trickle/p6.bin": route_trickle_page,
         "/trickle/p7.bin": route_trickle_page,
+        "/dcancel/index.html": route_dcancel_index,
+        "/dcancel/p0.bin": route_dcancel_page,
+        "/dcancel/p1.bin": route_dcancel_page,
+        "/dcancel/p2.bin": route_dcancel_page,
+        "/dcancel/p3.bin": route_dcancel_page,
+        "/dcancel/p4.bin": route_dcancel_page,
+        "/dcancel/p5.bin": route_dcancel_page,
+        "/dcancel/p6.bin": route_dcancel_page,
+        "/dcancel/p7.bin": route_dcancel_page,
         "/bigfiles/index.html": route_bigfiles_index,
         "/bigfiles/p0.bin": route_bigfile,
         "/bigfiles/p1.bin": route_bigfile,