htsserver never returns from main(), it blocks on its own exit wait

background_threads counted two unrelated things: the pinger, which only ever
leaves through exit(), and each mirror thread, which does return. main() then
waited on background_threads - 1, so with no pinger it asked for a negative
number of outstanding threads and the counter cannot go there, and with one it
waited on the pinger itself. Either way htsserver hung instead of returning.

Count only what never returns, and let each wait name what it must leave
running: the pinger for main(), the pinger plus itself for back_launch_cmd().
Waiting on background_threads alone would have swapped the hang for an early
return, letting main() tear down while a mirror was still crawling.

Closes #753

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <xroche@gmail.com>

configure.ac

@@ -173,10 +173,7 @@ AC_CHECK_TYPE(sa_family_t, [], [AC_DEFINE([sa_family_t], [uint16_t], [sa_family_
 AX_CHECK_ALIGNED_ACCESS_REQUIRED
 
 # check for various headers
-AC_CHECK_HEADERS([execinfo.h sys/ioctl.h sys/random.h])
-
-## CSPRNG for the --single-file mark; /dev/urandom is the fallback
-AC_CHECK_FUNCS([getrandom])
+AC_CHECK_HEADERS([execinfo.h sys/ioctl.h])
 
 ### zlib (mandatory)
 CHECK_ZLIB()

fuzz/corpus/singlefile/img-src.html

@@ -1,4 +1,3 @@
-<img src="a.png#!htsinline">
-<img src=big.png#!htsinline alt=over-cap>
-<img src="../escape.png#!htsinline"><img src="/abs.png#!htsinline">
-<img src="data:,x#!htsinline">
+<img src="a.png">
+<img src=big.png alt=over-cap>
+<img src="../escape.png"><img src="/abs.png"><img src="data:,x">

fuzz/corpus/singlefile/link-rel.html

@@ -1,3 +1,4 @@
-<link rel="stylesheet" href="s.css#!htsinline">
-<link rel="icon" href="a.png#!htsinline">
-<link rel="canonical" href="other.html#!htsinline">
+<link rel="stylesheet" href="s.css">
+<link rel=icon href=a.png>
+<link rel="next" href="p2.html">
+<link rel="preload" href="j.js">

fuzz/corpus/singlefile/malformed.html

@@ -1,3 +1,5 @@
-<p a=1 b=2 c=3 d=4 e=5 f=6 g=7 h=8 i=9 j=10 title="> <img src=a.png#!htsinline>">x</p>
-<!--><img src="a.png#!htsinline">
-<img src="a.png#!htsinline" >
+<img src="unterminated.png
+<div style="background:url(a.png">
+<style>@import url(
+<!-- unterminated comment
+<a href=

fuzz/corpus/singlefile/many-attrs.html

@@ -0,0 +1 @@
+<img src="a.png" a0="v" a1="v" a2="v" a3="v" a4="v" a5="v" a6="v" a7="v" a8="v" a9="v" a10="v" a11="v" a12="v" a13="v" a14="v" a15="v" a16="v" a17="v" a18="v" a19="v" a20="v" a21="v" a22="v" a23="v" a24="v" a25="v" a26="v" a27="v" a28="v" a29="v" a30="v" a31="v" a32="v" a33="v" a34="v" a35="v" a36="v" a37="v" a38="v" a39="v" a40="v" a41="v" a42="v" a43="v" a44="v" a45="v" a46="v" a47="v" a48="v" a49="v" a50="v" a51="v" a52="v" a53="v" a54="v" a55="v" a56="v" a57="v" a58="v" a59="v" a60="v" a61="v" a62="v" a63="v" a64="v" a65="v" a66="v" a67="v" a68="v" a69="v">

fuzz/corpus/singlefile/mark-at-eof.html

@@ -1 +0,0 @@
-a.png#!htsinline

fuzz/corpus/singlefile/mark-degenerate.html

@@ -1,5 +0,0 @@
-#!htsinline#!htsinline#!htsinline
-=#!htsinline=
-"#!htsinline"
-a.png#!htsinlin
-#!htsinlin

fuzz/corpus/singlefile/mark-only.html

@@ -1 +0,0 @@
-#!htsinline

fuzz/corpus/singlefile/rawtext.html

@@ -1,3 +1,4 @@
-<script src="j.js#!htsinline"></script>
-<script>var u="a.png#!htsinline";</script>
-<textarea>a.png#!htsinline</textarea>
+<script>var s="</scripting>"; if(a</b) x=1;</script>
+<script src="j.js"></script>
+<textarea></textareas></textarea>
+<title></titles></title>

fuzz/corpus/singlefile/srcset.html

@@ -1 +1,2 @@
-<img srcset="a.png#!htsinline 1x, big.png#!htsinline 2x, a.png#!htsinline 480w">
+<img srcset="a.png 1x, big.png 2x, a.png 100w">
+<source srcset="a.png,, a.png 2x," src="a.png">

fuzz/corpus/singlefile/style-attr.html

@@ -1,2 +1,2 @@
-<div style="background:url(a.png#!htsinline)"></div>
-<div style='background:url("a.png#!htsinline")'></div>
+<div style="background:url(a.png);list-style:url('a.png')"></div>
+<p style='background:url("a.png")'>x</p>

fuzz/corpus/singlefile/style-block.html

@@ -1,4 +1,5 @@
-<style>@import "s.css#!htsinline";
-@import url(sub/b.css#!htsinline);
-div{background:url(a.png#!htsinline)}
+<style>@import "s.css";
+@import url(sub/b.css);
+div{background:url(a.png)}
+/* url(a.png) */ p:after{content:"url(a.png)"}
 </style>

fuzz/corpus/singlefile/traversal.html

@@ -1,4 +0,0 @@
-<img src="../../../../etc/passwd#!htsinline">
-<img src="a.png#!htsinline#frag?q=1">
-<img src="sub/../a.png#!htsinline">
-<img src="./a.png#!htsinline">

fuzz/fuzz-singlefile.c

@@ -95,13 +95,10 @@ static void sf_init(void) {
   sf_write("a.png", png, sizeof(png) - 1);
   sf_write("big.png", big, sizeof(big));
   sf_text("j.js", "var x=1;\n");
-  /* Marked, so an inlined stylesheet recurses into its own marks and its
-     un-inlinable reference is rebased; unmarked assets leave the target as a
-     bare scan that reaches nothing. */
-  sf_text("s.css", "@import url(sub/b.css" SINGLEFILE_MARK ");\n"
-                   "div{background:url(a.png" SINGLEFILE_MARK ")}\n"
-                   "p{background:url(big.png" SINGLEFILE_MARK ")}\n");
-  sf_text("sub/b.css", "p{background:url(../a.png" SINGLEFILE_MARK ")}\n");
+  /* @import plus a url(), so an inlined stylesheet recurses and its own
+     relative reference is rebased. */
+  sf_text("s.css", "@import url(sub/b.css);\ndiv{background:url(a.png)}\n");
+  sf_text("sub/b.css", "p{background:url(../a.png)}\n");
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {

src/htsback.c

@@ -585,6 +585,14 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   return 0;
 }
 
+/* Move src onto dst; RENAME does not clobber an existing target on Windows. */
+static hts_boolean replace_file(const char *src, const char *dst) {
+  if (RENAME(src, dst) == 0)
+    return HTS_TRUE;
+  (void) UNLINK(dst);
+  return RENAME(src, dst) == 0 ? HTS_TRUE : HTS_FALSE;
+}
+
 /* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
    existing file moved the good copy to back->tmpfile before truncating url_sav.
    commit keeps the new file and drops the backup; else restore it so an aborted
@@ -602,7 +610,7 @@ static void back_finalize_backup(httrackp *opt, lien_back *const back,
     }
     /* On failure keep the backup: an orphaned temp beats losing the good copy.
      */
-    if (!hts_rename_over(back->tmpfile, back->url_sav))
+    if (!replace_file(back->tmpfile, back->url_sav))
       hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
                     "could not restore %s; previous copy kept as %s",
                     back->url_sav, back->tmpfile);
@@ -733,7 +741,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                                  "Read error when decompressing");
                     }
                     UNLINK(unpacked);
-                  } else if (hts_rename_over(unpacked, back[p].url_sav)) {
+                  } else if (replace_file(unpacked, back[p].url_sav)) {
                     /* The temp bypassed filecreate(), which is what chmods. */
 #ifndef _WIN32
                     chmod(back[p].url_sav, HTS_ACCESS_FILE);

src/htscoremain.c

@@ -1221,16 +1221,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   opt->mimehtml = 0;
                   com++;
                 }
-                if (opt->mimehtml && opt->single_file) {
-                  HTS_PANIC_PRINTF(
-                      "-%M and --single-file are two ways to make one "
-                      "self-contained file, so pick one: MIME (-%M) carries "
-                      "text parts without the base64 tax and stores a shared "
-                      "asset once; single-file HTML opens anywhere by "
-                      "double-click.");
-                  htsmain_free();
-                  return -1;
-                }
                 break;
               case 'k':
                 opt->nokeepalive = 0;
@@ -1842,16 +1832,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     com++;
                   }
                 }
-                if (opt->single_file && opt->mimehtml) {
-                  HTS_PANIC_PRINTF(
-                      "-%M and --single-file are two ways to make one "
-                      "self-contained file, so pick one: MIME (-%M) carries "
-                      "text parts without the base64 tax and stores a shared "
-                      "asset once; single-file HTML opens anywhere by "
-                      "double-click.");
-                  htsmain_free();
-                  return -1;
-                }
                 break;
               case 'm': // sitemap / sitemap-url: seed the crawl from sitemaps
                 if (*(com + 1) == 'u') { // --sitemap-url URL: explicit sitemap

src/htslib.c

@@ -6037,7 +6037,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->changes_state = NULL;
   opt->single_file = HTS_FALSE;
   opt->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
-  opt->singlefile_state = NULL;
   StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
@@ -6192,7 +6191,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->warc_file);
     StringFree(opt->sitemap_url);
     hts_sitemap_free(opt); /* backstop: httpmirror's early-return paths */
-    singlefile_free(opt);
 
     hts_changes_free_opt(opt);
 

src/htsopt.h

@@ -564,8 +564,6 @@ struct httrackp {
      rather than in htsoptstate because that struct is embedded by value, so
      growing it would shift every httrackp field declared after it. */
   void *sitemap_state; /**< hts_sitemap_state*, or NULL. Tail: ABI */
-  void *singlefile_state; /**< hts_singlefile_state*, or NULL; holds this run's
-                               mark secret, never persisted. Tail: ABI */
 };
 
 /* Running statistics for a mirror. */

src/htsparse.c

@@ -59,7 +59,6 @@ Please visit our Website: http://www.httrack.com
 
 // parser
 #include "htsparse.h"
-#include "htssinglefile.h"
 #include "htsback.h"
 
 // arrays
@@ -385,7 +384,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
     FILE *fp = NULL;                  // fichier écrit localement 
     const char *html = r->adr;        // pointeur (on parcours)
     const char *lastsaved;            // adresse du dernier octet sauvé + 1
-    hts_boolean sf_may_mark = HTS_FALSE; // --single-file: mark this one?
 
     hts_log_print(opt, LOG_DEBUG, "scanning file %s%s (%s)..", urladr(), urlfil(),
                   savename());
@@ -401,13 +399,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           r->adr[i] = ' ';
         }
       }
-      /* --single-file: RFC 2046 makes the generator responsible for a
-         boundary that cannot collide with what it encapsulates. Same duty
-         here, and the same negligible odds: only chance can put this run's
-         secret in a fetched document, since no site can spell it. */
-      if (opt->single_file) {
-        sf_may_mark = singlefile_may_mark(opt, r->adr, (size_t) r->size);
-      }
     }
 
     // Indexing!
@@ -3052,32 +3043,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                         if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
 
-                          /* --single-file: tag the reference for the
-                             end-of-mirror pass. A fragment, so a mirror left
-                             marked by an interrupted run still browses. */
-                          const char sf_class =
-                              (opt->single_file && sf_may_mark && !in_media &&
-                               p_type == 0 && !p_searchMETAURL)
-                                  ? singlefile_ref_class(
-                                        intag_start_valid ? intag_name : NULL,
-                                        tag_attr_start)
-                                  : 0;
-                          const size_t sf_start = TypedArraySize(output_buffer);
-
                           // écrire le lien modifié, relatif
                           // Note: escape all chars, even >127 (no UTF)
                           HT_ADD_HTMLESCAPED_FULL(tempo);
 
-                          /* Measured on what was appended, not on tempo: the
-                             escape above is free to change the length. */
-                          if (sf_class != 0) {
-                            char sf_mark[SINGLEFILE_MARK_MAX];
-
-                            HT_ADD(singlefile_mark(
-                                opt, sf_mark, sizeof(sf_mark), sf_class,
-                                TypedArraySize(output_buffer) - sf_start));
-                          }
-
                           // Add query-string, for informational purpose only
                           // Useless, because all parameters-pages are saved into different targets
                           if (opt->includequery) {

src/htsselftest.c

@@ -5201,47 +5201,11 @@ static hts_boolean sf_try_put(const char *dir, const char *rel,
   return HTS_TRUE;
 }
 
-/* Expand a fixture: \001<ref>\002 becomes <ref> plus this run's mark for it.
-   The secret is random per run, so a fixture cannot spell a mark itself. */
-static void sf_expand_fixture(httrackp *opt, const char *in, size_t len,
-                              String *out) {
-  size_t i, start = 0;
-
-  StringClear(*out);
-  for (i = 0; i < len; i++) {
-    if (in[i] == '\001') {
-      start = StringLength(*out);
-    } else if (in[i] == '\002') {
-      char mark[SINGLEFILE_MARK_MAX];
-
-      StringCat(*out,
-                singlefile_mark(opt, mark, sizeof(mark), SINGLEFILE_CLASS_ANY,
-                                StringLength(*out) - start));
-    } else {
-      StringAddchar(*out, in[i]);
-    }
-  }
-}
-
 static void sf_put(const char *dir, const char *rel, const void *data,
                    size_t len) {
   assertf(sf_try_put(dir, rel, data, len));
 }
 
-/* sf_put() for a text fixture, expanding its \001<ref>\002 delimiters. Binary
-   fixtures must not go through here: sf_png carries those very bytes. */
-static size_t sf_put_marked(httrackp *opt, const char *dir, const char *rel,
-                            const void *data, size_t len) {
-  String body = STRING_EMPTY;
-  size_t written;
-
-  sf_expand_fixture(opt, (const char *) data, len, &body);
-  assertf(sf_try_put(dir, rel, StringBuff(body), StringLength(body)));
-  written = StringLength(body);
-  StringFree(body);
-  return written;
-}
-
 /* Number of times needle occurs in hay. */
 static int sf_count(const char *hay, const char *needle) {
   const size_t l = strlen(needle);
@@ -5351,80 +5315,75 @@ static const char sf_png[] = "\x89PNG\r\n\x1a\n\x00\x01\x02\xff";
 
 static const char sf_page[] =
     "<html><head>\n"
-    "<link rel=\"stylesheet\" href=\"\001css/main.css\002\">\n"
-    "<link rel=\"canonical\" href=\"\001other.html\002\">\n"
+    "<link rel=\"stylesheet\" href=\"css/main.css\">\n"
+    "<link rel=\"canonical\" href=\"other.html\">\n"
     "<title>t</title>\n"
-    "<style>body { background: url(\"\001img/a%20b.png\002\"); }</style>\n"
+    "<style>body { background: url(\"img/a%20b.png\"); }</style>\n"
     "</head><body>\n"
-    "<img src=\"\001img/a%20b.png\002\" srcset=\"\001img/a%20b.png\002 "
-    "1x, \001img/big.png\002 2x\">\n"
-    "<link rel=\"icon\" href=\"\001icon.png\002\">\n"
-    "<link rel=\"preload\" as=\"font\" href=\"\001font/f.woff2\002\">\n"
+    "<img src=\"img/a%20b.png\" srcset=\"img/a%20b.png 1x, img/big.png 2x\">\n"
+    "<link rel=\"icon\" href=\"icon.png\">\n"
+    "<link rel=\"preload\" as=\"font\" href=\"font/f.woff2\">\n"
     "<img src=\"data:image/gif;base64,QUJD\">\n"
     /* Each has a real file where its guard's removal would land it; without
        that they stay links either way, the target merely being absent. */
     "<img src=\"http://example.com/x.png\">\n"
     "<img src=\"//example.com/x.png\">\n"
-    "<input type=\"image\" src=\"\001img/in.png\002\">\n"
+    "<input type=\"image\" src=\"img/in.png\">\n"
     /* Lazy loading: src is the placeholder, the real image rides data-src. */
-    "<img src=\"\001img/ph.png\002\" data-src=\"\001img/lz.png\002\" "
-    "data-srcset=\"\001img/lz2.png\002 2x\" "
-    "lowsrc=\"\001img/low.png\002\">\n"
-    "<object data=\"\001img/ob.png\002\"></object>\n"
-    "<embed src=\"\001img/em.png\002\">\n"
-    "<img data-src=\"\001other.html\002\">\n"
+    "<img src=\"img/ph.png\" data-src=\"img/lz.png\" "
+    "data-srcset=\"img/lz2.png 2x\" lowsrc=\"img/low.png\">\n"
+    "<object data=\"img/ob.png\"></object>\n"
+    "<embed src=\"img/em.png\">\n"
+    "<img data-src=\"other.html\">\n"
     /* What a first pass emits: re-resolving it is what a second pass must not
        do, and the fallback type would inline whatever the walk found. */
     "<link rel=\"stylesheet\" href=\"data:text/css;base64,QUJD\">\n"
-    "<video poster=\"\001img/po.png\002\" controls>"
-    "<source src=\"\001v.mp4\002\" type=\"video/mp4\"></video>\n"
-    "<svg><image href=\"\001img/sv.png\002\"/></svg>\n"
-    "<table background=\"\001img/bg.png\002\"><tr><td>x</td></tr></table>\n"
+    "<video poster=\"img/po.png\" controls>"
+    "<source src=\"v.mp4\" type=\"video/mp4\"></video>\n"
+    "<svg><image href=\"img/sv.png\"/></svg>\n"
+    "<table background=\"img/bg.png\"><tr><td>x</td></tr></table>\n"
     /* The second is what bites: drop the clamp and its leading ".." lands it
        back on <root>/img/a b.png. The first can only 404 either way. */
-    "<img src=\"\001../escape.png\002\">\n"
-    "<img src=\"\001../img/a%20b.png\002\">\n"
+    "<img src=\"../escape.png\">\n"
+    "<img src=\"../img/a%20b.png\">\n"
     "<a href=\"img/a%20b.png\">link</a>\n"
-    "<script src=\"\001js/app.js\002\"></script>\n"
+    "<script src=\"js/app.js\"></script>\n"
     "<script>var s = \"</scripting>\"; var t = \"<img src='img/a%20b.png'>\";"
     "</script>\n"
-    "<img src=\"\001missing.png\002\" >\n"
-    "<!--><img src=\"\001img/a%20b.png\002\">\n"
-    "<div style=\"background:url(\001img/a%20b.png\002)\"></div>\n"
-    "<div style='content:\"x\"; "
-    "background:url(\001img/a%20b.png\002)'></div>\n"
+    "<img src=\"missing.png\" >\n"
+    "<!--><img src=\"img/a%20b.png\">\n"
+    "<div style=\"background:url(img/a%20b.png)\"></div>\n"
+    "<div style='content:\"x\"; background:url(img/a%20b.png)'></div>\n"
     "</body></html>\n";
 
 /* Lay a small mirror down under root. */
-static void sf_fixture(httrackp *opt, const char *root) {
+static void sf_fixture(const char *root) {
   /* The over-cap url() is what drives the rebase fallback: a reference an
      inlined stylesheet could not embed has to come out relative to the page,
      not to the stylesheet, or it dangles. */
   static const char css[] =
-      "@import \"\001sub/nested.css\002\";\n"
-      "@import url(\"\001sub/two.css\002\");\n"
+      "@import \"sub/nested.css\";\n"
+      "@import url(\"sub/two.css\");\n"
       "@import \"a\\\"url(../img/a b.png)b.css\";\n"
-      "@font-face { font-family: f; src: url(\001../font/f.woff2\002); }\n"
-      "body { background: url(\001../img/a%20b.png\002); }\n"
-      "div { background: url(\001../img/big.png\002); }\n"
+      "@font-face { font-family: f; src: url(../font/f.woff2); }\n"
+      "body { background: url(../img/a b.png); }\n"
+      "div { background: url(../img/big.png); }\n"
       "/* url(../img/never.png) */\n";
-  static const char nested[] =
-      "div { background: url(\001../../img/a%20b.png\002); }\n";
-  static const char two[] =
-      "p { background: url(\001../../img/a%20b.png\002); }\n";
-  static const char deep[] = "<html><head><link rel=\"stylesheet\" "
-                             "href=\"\001../../css/main.css\002\">\n"
-                             "</head><body>d</body></html>\n";
+  static const char nested[] = "div { background: url(../../img/a b.png); }\n";
+  static const char two[] = "p { background: url(../../img/a b.png); }\n";
+  static const char deep[] =
+      "<html><head><link rel=\"stylesheet\" href=\"../../css/main.css\">\n"
+      "</head><body>d</body></html>\n";
   static const char js[] = "var app = 1;\n";
   char big[4096];
 
   memset(big, 'B', sizeof(big));
-  sf_put_marked(opt, root, "page.html", sf_page, sizeof(sf_page) - 1);
-  sf_put_marked(opt, root, "deep/sub/page.html", deep, sizeof(deep) - 1);
+  sf_put(root, "page.html", sf_page, sizeof(sf_page) - 1);
+  sf_put(root, "deep/sub/page.html", deep, sizeof(deep) - 1);
   sf_put(root, "other.html", "<html>o</html>", 14);
-  sf_put_marked(opt, root, "css/main.css", css, sizeof(css) - 1);
-  sf_put_marked(opt, root, "css/sub/nested.css", nested, sizeof(nested) - 1);
-  sf_put_marked(opt, root, "css/sub/two.css", two, sizeof(two) - 1);
+  sf_put(root, "css/main.css", css, sizeof(css) - 1);
+  sf_put(root, "css/sub/nested.css", nested, sizeof(nested) - 1);
+  sf_put(root, "css/sub/two.css", two, sizeof(two) - 1);
   sf_put(root, "js/app.js", js, sizeof(js) - 1);
   sf_put(root, "img/a b.png", sf_png, SF_PNG_LEN);
   sf_put(root, "img/big.png", big, sizeof(big));
@@ -5462,7 +5421,7 @@ static void sf_fixture(httrackp *opt, const char *root) {
     }
     StringCat(wide,
               " title=\"> <img src=img/a%20b.png> \">end</p></body></html>");
-    sf_put_marked(opt, root, "wide.html", StringBuff(wide), StringLength(wide));
+    sf_put(root, "wide.html", StringBuff(wide), StringLength(wide));
     StringFree(wide);
   }
   sf_put(root, "v.mp4",
@@ -5488,7 +5447,7 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
   sf_put(argv[0], "escape.png", sf_png,
          SF_PNG_LEN); /* just outside the mirror */
   fconcat(root, sizeof(root), argv[0], "mirror/");
-  sf_fixture(opt, root);
+  sf_fixture(root);
   fconcat(page, sizeof(page), root, "page.html");
 
   /* Cap between the small assets and big.png. */
@@ -5556,10 +5515,13 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
   sf_check(strstr(out, "var t = \"<img src='img/a%20b.png'>\";") != NULL,
            "script body rewritten past a </scripting> lookalike");
 
-  /* Only the marked reference is touched: the value keeps its own quoting, so
-     nothing can be emitted that the attribute could not already hold. */
-  sf_check(strstr(out, "style='content:\"x\"; background:url(data:") != NULL,
-           "style attribute re-quoted instead of substituted in place");
+  /* Nothing an attribute value cannot hold: url() stays unquoted, and a quote
+     that was already in the CSS is escaped. */
+  sf_check(strstr(out, "url(\"data:") == NULL,
+           "a quoted url() would end a style attribute");
+  sf_check(strstr(out, "style=\"content:&quot;x&quot;; background:url(data:") !=
+               NULL,
+           "quote inside a rewritten style attribute not escaped");
 
   sf_check(strstr(out, "img/big.png 2x") != NULL, "over-cap asset inlined");
   sf_check(strstr(out, " 1x") != NULL, "srcset descriptor lost");
@@ -5579,7 +5541,7 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
              "url() inside a CSS comment was rewritten");
     sf_check(strstr(css, "url(data:font/woff2;base64,") != NULL,
              "@font-face src not inlined");
-    sf_check(strstr(css, "@import url(\"data:text/css;base64,") != NULL,
+    sf_check(strstr(css, "@import url(data:text/css;base64,") != NULL,
              "@import url() form not inlined");
     sf_check(strstr(css, "url(../img/a b.png)b.css") != NULL,
              "url() inside a string with an escaped quote was rewritten");
@@ -5643,7 +5605,7 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
 
   /* Same page, a cap above big.png: it now inlines. */
   fconcat(root, sizeof(root), argv[0], "mirror2/");
-  sf_fixture(opt, root);
+  sf_fixture(root);
   fconcat(page, sizeof(page), root, "page.html");
   opt->single_file_max_size = 1024 * 1024;
   sf_check(singlefile_rewrite_file(opt, root, page),
@@ -5657,37 +5619,21 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
   /* Nothing inlines, so the rewriter must be byte transparent. Assert on its
      output: the file it declined to write could not have changed regardless. */
   fconcat(root, sizeof(root), argv[0], "mirror3/");
-  sf_fixture(opt, root);
+  sf_fixture(root);
   fconcat(page, sizeof(page), root, "page.html");
   opt->single_file_max_size = 1;
-  /* The page is still rewritten: nothing inlines, but the marks must go. */
-  sf_check(singlefile_rewrite_file(opt, root, page),
-           "one-byte cap left the marks in place");
-  {
-    char *capped = readfile_utf8(page);
-
-    /* Only the two data: URIs the fixture itself ships. */
-    sf_check(capped != NULL && sf_count(capped, ";base64,") == 2,
-             "one-byte cap inlined");
-    freet(capped);
-  }
+  sf_check(!singlefile_rewrite_file(opt, root, page), "one-byte cap inlined");
   {
     String verbatim = STRING_EMPTY;
 
     StringClear(verbatim);
-    String marked = STRING_EMPTY;
-
-    sf_expand_fixture(opt, sf_page, sizeof(sf_page) - 1, &marked);
-    (void) singlefile_rewrite_html(opt, root, page, StringBuff(marked),
-                                   StringLength(marked),
+    (void) singlefile_rewrite_html(opt, root, page, sf_page,
+                                   sizeof(sf_page) - 1,
                                    SINGLEFILE_MAX_PAGE_SIZE, &verbatim);
-    /* Mark-transparent, not byte-transparent: a reference that cannot be
-       inlined loses its mark and keeps everything else. */
-    sf_check(StringLength(verbatim) < StringLength(marked),
-             "a page with nothing to inline kept its marks");
-    sf_check(strstr(StringBuff(verbatim), singlefile_intro(opt)) == NULL,
-             "an un-inlinable reference kept its mark");
-    StringFree(marked);
+    sf_check(StringLength(verbatim) == sizeof(sf_page) - 1 &&
+                 memcmp(StringBuff(verbatim), sf_page, sizeof(sf_page) - 1) ==
+                     0,
+             "a page with nothing to inline was re-serialized differently");
     StringFree(verbatim);
   }
   (void) outlen;
@@ -5696,29 +5642,26 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
      run is the control: it proves the fan-out is real, so the small one was
      cut short by the budget and not by the fixture. */
   {
-    static const char bomb_css[] =
-        "@import \"\001b.css\002\";@import \"\001b.css\002\";"
-        "@import \"\001b.css\002\";@import \"\001b.css\002\";\n";
-    static const char bomb_html[] =
-        "<html><head>"
-        "<link rel=\"stylesheet\" href=\"\001b.css\002\">"
-        "</head></html>\n";
-    size_t css_len;
-    String small = STRING_EMPTY, large = STRING_EMPTY, bomb = STRING_EMPTY;
+    static const char bomb_css[] = "@import \"b.css\";@import \"b.css\";"
+                                   "@import \"b.css\";@import \"b.css\";\n";
+    static const char bomb_html[] = "<html><head>"
+                                    "<link rel=\"stylesheet\" href=\"b.css\">"
+                                    "</head></html>\n";
+    const size_t css_len = sizeof(bomb_css) - 1;
+    String small = STRING_EMPTY, large = STRING_EMPTY;
 
     fconcat(root, sizeof(root), argv[0], "bomb/");
-    css_len = sf_put_marked(opt, root, "b.css", bomb_css, sizeof(bomb_css) - 1);
+    sf_put(root, "b.css", bomb_css, css_len);
     fconcat(page, sizeof(page), root, "page.html");
     opt->single_file_max_size = 1024 * 1024;
     StringClear(small);
     StringClear(large);
-    sf_expand_fixture(opt, bomb_html, sizeof(bomb_html) - 1, &bomb);
-    (void) singlefile_rewrite_html(opt, root, page, StringBuff(bomb),
-                                   StringLength(bomb), (LLint) css_len * 3,
+    (void) singlefile_rewrite_html(opt, root, page, bomb_html,
+                                   sizeof(bomb_html) - 1, (LLint) css_len * 3,
                                    &small);
-    (void) singlefile_rewrite_html(opt, root, page, StringBuff(bomb),
-                                   StringLength(bomb), SINGLEFILE_MAX_PAGE_SIZE,
-                                   &large);
+    (void) singlefile_rewrite_html(opt, root, page, bomb_html,
+                                   sizeof(bomb_html) - 1,
+                                   SINGLEFILE_MAX_PAGE_SIZE, &large);
     sf_check(StringLength(large) > 4096, "the @import bomb did not fan out");
     sf_check(StringLength(small) < StringLength(large) / 8,
              "the per-page budget did not cut the fan-out short");
@@ -5728,7 +5671,6 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
              "the per-page budget was not charged as each asset was taken");
     StringFree(small);
     StringFree(large);
-    StringFree(bomb);
   }
 
   if (!sf_colon_ok)

src/htssinglefile.h

@@ -55,62 +55,18 @@ extern "C" {
    so a few hundred bytes of hostile CSS can otherwise ask for gigabytes. */
 #define SINGLEFILE_MAX_PAGE_SIZE (64 * 1024 * 1024)
 
-/* The mark htsparse appends to a saved reference the pass may inline:
-
-     #!<16-hex secret>.<class>.<len>
-
-   A fragment, so a mirror left marked by an interrupted run still browses.
-   The secret is 64 CSPRNG bits drawn once per run and never written to disk,
-   which is what makes the mark unforgeable: a site cannot spell one, so no
-   sanitiser has to keep hostile bytes away from it. <len> is the byte length
-   of the reference text immediately preceding the mark, so the pass never has
-   to guess where that reference starts. <class> is the context htsparse saw,
-   checked against the resolved type so a mismatch fails loudly. */
-#define SINGLEFILE_MARK_INTRO "#!"
-#define SINGLEFILE_SECRET_HEX 16
-
-/* <class>: what the referencing context expects. */
-#define SINGLEFILE_CLASS_ANY '-'
-#define SINGLEFILE_CLASS_CSS 'c'
-#define SINGLEFILE_CLASS_JS 'j'
-
-/* Enough for the intro, the secret, both separators and a 20-digit length. */
-#define SINGLEFILE_MARK_MAX 64
-
-/* Release this run's secret. */
-void singlefile_free(httrackp *opt);
-
-/* "#!<secret>" for this run, or NULL if no CSPRNG was available (in which case
-   nothing may be marked). Draws the secret on first use. */
-const char *singlefile_intro(httrackp *opt);
-
-/* HTS_FALSE if [body,len) already contains this run's intro, in which case the
-   document must not be marked: RFC 2046 gives the generator the same duty for
-   a MIME boundary. Only a 2^-64 coincidence can trip it, since the intro is
-   not something fetched content can spell. */
-hts_boolean singlefile_may_mark(httrackp *opt, const char *body, size_t len);
-
-/* Write the mark for a reference of reflen bytes into buf; returns buf. */
-const char *singlefile_mark(httrackp *opt, char *buf, size_t bufsize, char cls,
-                            size_t reflen);
-
-/* The class a reference in this context may inline as, or 0 to leave it alone.
-   tag_name points just past the '<' of the enclosing start tag, or NULL when
-   there is none (inside a stylesheet or a script); attr at the attribute name.
-   Everything htsparse detects is inlinable unless it names a page. */
-char singlefile_ref_class(const char *tag_name, const char *attr);
-
-/* Rewrite every HTML page the mirror produced, then strip the marks left in
-   the assets. No-op unless opt->single_file; call once the tree is final,
-   after the update purge. */
+/* Rewrite every HTML page the mirror produced. No-op unless opt->single_file;
+   call once the tree is final, after the update purge. */
 void singlefile_process_mirror(httrackp *opt);
 
-/* Expand the marks in the document held in memory, appending the result to
-   out. root is the mirror directory that references may not escape; page_path
-   is the document's own path under it (both UTF-8, '/' or native separators).
-   page_budget caps the total inlined bytes, since a nested @import fans out
+/* Rewrite one HTML document held in memory, appending the result to out.
+   root is the mirror directory that references may not escape; page_path is
+   the document's own path under it (both UTF-8, '/' or native separators).
+   page_budget caps the total inlined bytes, since nested @import fans out
    multiplicatively; the mirror pass passes SINGLEFILE_MAX_PAGE_SIZE.
-   Returns HTS_TRUE if at least one reference was replaced. */
+   Returns HTS_TRUE if at least one reference was replaced; out may still
+   differ from the input when that is HTS_FALSE, since a style or srcset value
+   is re-serialized in place. */
 hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
                                     const char *page_path, const char *html,
                                     size_t html_len, LLint page_budget,

src/htstools.c

@@ -41,14 +41,8 @@ Please visit our Website: http://www.httrack.com
 #include "htsstrings.h"
 #include "htscharset.h"
 #ifdef _WIN32
-/* before <stdlib.h>, which is what declares rand_s under it */
-#define _CRT_RAND_S
 #include "windows.h"
 #else
-#include <errno.h>
-#ifdef HAVE_SYS_RANDOM_H
-#include <sys/random.h>
-#endif
 #include <dirent.h>
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -1449,59 +1443,3 @@ HTSEXT_API hts_boolean hts_findissystem(find_handle find) {
   }
   return 0;
 }
-
-hts_boolean hts_random_bytes(void *buf, size_t len) {
-  unsigned char *const p = (unsigned char *) buf;
-
-#ifdef _WIN32
-  size_t i;
-
-  /* rand_s() is the CRT's wrapper over the system CSPRNG, and unlike
-     BCryptGenRandom it needs no extra import library. */
-  for (i = 0; i < len; i += sizeof(unsigned int)) {
-    const size_t left = len - i;
-    unsigned int v;
-
-    if (rand_s(&v) != 0)
-      return HTS_FALSE;
-    memcpy(p + i, &v, left < sizeof(v) ? left : sizeof(v));
-  }
-  return HTS_TRUE;
-#else
-  size_t got = 0;
-  FILE *fp;
-
-#ifdef HAVE_GETRANDOM
-  while (got < len) {
-    const ssize_t n = getrandom(p + got, len - got, 0);
-
-    if (n < 0) {
-      if (errno == EINTR)
-        continue;
-      break; /* pre-3.17 kernel or a seccomp filter: try /dev/urandom */
-    }
-    got += (size_t) n;
-  }
-  if (got == len)
-    return HTS_TRUE;
-#endif
-  fp = fopen("/dev/urandom", "rb");
-  if (fp == NULL)
-    return HTS_FALSE;
-  got += fread(p + got, 1, len - got, fp);
-  fclose(fp);
-  return got == len ? HTS_TRUE : HTS_FALSE;
-#endif
-}
-
-hts_boolean hts_rename_over(const char *src, const char *dst) {
-  char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
-
-  fconv(csrc, sizeof(csrc), src);
-  fconv(cdst, sizeof(cdst), dst);
-  if (RENAME(csrc, cdst) == 0)
-    return HTS_TRUE;
-  /* RENAME does not clobber an existing target on Windows. */
-  (void) UNLINK(cdst);
-  return RENAME(csrc, cdst) == 0 ? HTS_TRUE : HTS_FALSE;
-}

src/htstools.h

@@ -137,14 +137,6 @@ HTSEXT_API hts_boolean hts_findisdir(find_handle find);
 HTSEXT_API hts_boolean hts_findisfile(find_handle find);
 HTSEXT_API hts_boolean hts_findissystem(find_handle find);
 
-/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both
-   paths are fconv()'d. */
-hts_boolean hts_rename_over(const char *src, const char *dst);
-
-/* Fill buf with len cryptographically strong random bytes. HTS_FALSE when no
-   such source is available: the caller must fail, never fall back to rand(). */
-hts_boolean hts_random_bytes(void *buf, size_t len);
-
 #endif
 
 #endif

src/htswarc.c

@@ -964,6 +964,16 @@ static zipFile wacz_zip_open(const char *path) {
   return zipOpen2_64(path, 0 /*create*/, NULL, &ff);
 }
 
+/* Move src onto dst (UTF-8/Windows-safe); RENAME won't clobber on Windows, so
+   fall back to unlink+rename. Returns 0 on success. */
+static int wacz_rename_over(const char *src, const char *dst) {
+  char cs[CATBUFF_SIZE], cd[CATBUFF_SIZE];
+  if (RENAME(fconv(cs, sizeof(cs), src), fconv(cd, sizeof(cd), dst)) == 0)
+    return 0;
+  (void) UNLINK(fconv(cd, sizeof(cd), dst));
+  return RENAME(fconv(cs, sizeof(cs), src), fconv(cd, sizeof(cd), dst));
+}
+
 /* Package the segment(s) + .cdx + a generated pages.jsonl into <base>.wacz at
    crawl end (the archive file(s) and .cdx are already closed on disk). */
 static void warc_wacz_package(warc_writer *w) {
@@ -1082,7 +1092,7 @@ static void warc_wacz_package(warc_writer *w) {
     hts_log_print(w->opt, LOG_WARNING,
                   "WACZ: packaging failed, kept existing %s untouched",
                   waczpath);
-  } else if (!hts_rename_over(tmppath, waczpath)) {
+  } else if (wacz_rename_over(tmppath, waczpath) != 0) {
     (void) UNLINK(fconv(catbuff, sizeof(catbuff), tmppath));
     hts_log_print(w->opt, LOG_WARNING | LOG_ERRNO,
                   "WACZ: could not finalize %s", waczpath);

src/htsweb.c

@@ -101,8 +101,8 @@ static void htsweb_sig_brpipe(int code) {
   /* ignore */
 }
 
-/* Number of background threads */
-static int background_threads = 0;
+/* Threads that never return; no wait may count on them draining. */
+static int nonjoinable_threads = 0;
 
 /* Server/client ping handling */
 static htsmutex pingMutex = HTSMUTEX_INIT;
@@ -299,15 +299,19 @@ int main(int argc, char *argv[]) {
 
   /* pinger */
   if (parentPid > 0) {
-    hts_newthread(client_ping, (void *) (uintptr_t) parentPid);
-    background_threads++; /* Do not wait for this thread! */
+    if (hts_newthread(client_ping, (void *) (uintptr_t) parentPid) == 0) {
+#ifndef _WIN32
+      nonjoinable_threads++; /* client_ping() only ever leaves through exit() */
+#endif
+    }
     smallserver_setpinghandler(pingHandler, NULL);
   }
 
   /* launch */
   ret = help_server(argv[1], defaultPort, bindAddr);
 
-  htsthread_wait_n(background_threads - 1);
+  /* Drain everything a mirror may still have in flight, the pinger aside. */
+  htsthread_wait_n(nonjoinable_threads);
   hts_uninit();
 
 #ifdef _WIN32
@@ -382,7 +386,6 @@ void webhttrack_main(char *cmd) {
   commandRunning = 1;
   DEBUG(fprintf(stderr, "commandRunning=1\n"));
   hts_newthread(back_launch_cmd, (void *) strdup(cmd));
-  background_threads++; /* Do not wait for this thread! */
 }
 
 void webhttrack_lock(void) {
@@ -423,8 +426,8 @@ static int webhttrack_runmain(httrackp * opt, int argc, char **argv) {
   /* Rock'in! */
   ret = hts_main2(argc, argv, opt);
 
-  /* Wait for pending threads to finish */
-  htsthread_wait_n(background_threads);
+  /* Wait for pending threads to finish; the pinger and this thread stay. */
+  htsthread_wait_n(nonjoinable_threads + 1);
 
   return ret;
 }

src/htswizard.c

@@ -61,6 +61,11 @@ Please visit our Website: http://www.httrack.com
     assertf((*opt->filters.filptr) < opt->maxfilter);                          \
   } while (0)
 
+typedef struct htspair_t {
+  const char *tag;
+  const char *attr;
+} htspair_t;
+
 /* "embedded" */
 htspair_t hts_detect_embed[] = {
   {"img", "src"},
@@ -128,13 +133,11 @@ int hts_acceptlink(httrackp * opt, int ptr,
   return forbidden_url;
 }
 
-hts_boolean hts_cmp_tag_token(const char *tag, const char *cmp) {
+static int cmp_token(const char *tag, const char *cmp) {
   int p;
 
-  return (tag != NULL && strncasecmp(tag, cmp, (p = (int) strlen(cmp))) == 0 &&
-          !isalnum((unsigned char) tag[p]))
-             ? HTS_TRUE
-             : HTS_FALSE;
+  return (strncasecmp(tag, cmp, (p = (int) strlen(cmp))) == 0
+          && !isalnum((unsigned char) tag[p]));
 }
 
 /* TRUE if (tag, attribute) matches an embedded-asset pair in the table */
@@ -142,8 +145,7 @@ static hts_boolean is_embed_pair(const htspair_t *table, const char *tag,
                                  const char *attribute) {
   int i;
   for (i = 0; table[i].tag != NULL; i++) {
-    if (hts_cmp_tag_token(tag, table[i].tag) &&
-        hts_cmp_tag_token(attribute, table[i].attr))
+    if (cmp_token(tag, table[i].tag) && cmp_token(attribute, table[i].attr))
       return HTS_TRUE;
   }
   return HTS_FALSE;

src/htswizard.h

@@ -56,19 +56,6 @@ hts_boolean hts_robots_forbids(httrackp *opt, const char *adr, const char *fil,
                                hts_boolean filters_decided,
                                hts_boolean filters_refused);
 
-/* A (tag, attribute) pair naming a reference kind. */
-#ifndef HTS_DEF_DEFSTRUCT_htspair_t
-#define HTS_DEF_DEFSTRUCT_htspair_t
-
-typedef struct htspair_t {
-  const char *tag;
-  const char *attr;
-} htspair_t;
-#endif
-
-/* HTS_TRUE if tag starts with the whole token cmp; NULL tag never matches. */
-hts_boolean hts_cmp_tag_token(const char *tag, const char *cmp);
-
 int hts_acceptlink(httrackp * opt, int ptr,
                    const char *adr, const char *fil,
                    const char *tag, const char *attribute,

tests/43_local-update-truncate.test

@@ -18,7 +18,6 @@ set -euo pipefail
 bash "$top_srcdir/tests/local-crawl.sh" \
     --rerun-args '--update -M400000' \
     --log-found 'More than 400000 bytes have been transferred.. giving up' \
-    --log-not-found 'could not restore' \
     --found bigtrunc/slow.bin \
     --file-min-bytes bigtrunc/slow.bin 655360 \
     --file-min-bytes bigtrunc/fast.bin 655360 \

tests/65_port-siblings.test

@@ -79,6 +79,29 @@ done
 # 65535 is a valid port the old "< 65535" bound refused
 web_accepted 65535
 
+# --- htsserver returns from main() instead of blocking at exit -------------
+# The exit wait must exclude exactly the threads that never return (#753).
+# $tmp has no lang.def, so the server fails right after announcing and main()
+# reaches the wait on its own; rc, not EXITED, carries the verdict.
+
+web_exits() {
+    local rc=0
+
+    : >"$tmp/exit.log"
+    run_with_timeout 30 htsserver "$tmp" "$@" >"$tmp/exit.log" 2>&1 || rc=$?
+    grep -q "^EXITED" "$tmp/exit.log" ||
+        ! echo "FAIL: #753: htsserver ${*:-(no options)} never finished serving" || exit 1
+    # exactly 1, not merely "not 124": a crash or an assertf abort also escapes
+    # the wait, and would otherwise read as a pass
+    test "$rc" -eq 1 ||
+        ! echo "FAIL: #753: htsserver ${*:-(no options)} exited $rc, want 1" || exit 1
+}
+
+# no pinger: the excluded count went negative
+web_exits
+# a pinger, which never returns and so must stay excluded
+web_exits --ppid $$
+
 # --- proxytrack <proxy-addr:port> <ICP-addr:port> --------------------------
 # A bad argument falls through to the usage screen; it had no range check at
 # all, so 65616 quietly listened on port 80. A valid one binds and blocks.

tests/74_local-warc-wacz.test

@@ -5,16 +5,11 @@
 # real gate: STORE-mode entries, the fixed layout, recomputed sha256 digests and
 # the datapackage-digest chain (plus py-wacz/pywb when importable). The crawl
 # skips cleanly on a build without OpenSSL (no conformant SHA-256 -> no package).
-#
-# The cacheless second pass repackages over the .wacz the first one left: the
-# clobber, not the happy rename, is what breaks when a platform refuses to
-# rename onto an existing file (#726).
 
 set -eu
 
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --wacz-validate \
-    --rerun-args '-C0' --log-not-found 'could not finalize' \
     --found 'mini304/index.html' --found 'mini304/page.html' \
     httrack 'BASEURL/mini304/index.html' --warc-file warc-out --wacz

tests/94_local-single-file.test

@@ -37,11 +37,6 @@ cat >"${doc}/index.html" <<'EOF'
 <img src="pixel.svg" data-src="lazy.svg" alt="l">
 <script src="app.js"></script>
 <a href="other.html">other</a>
-<p>id=pixel.svg#!htsinline end</p>
-<link rel="stylesheet" href="style_noext">
-<script src="app_noext"></script>
-<img src="pixel.svg#icon">
-<a href="pixel.svg">raw</a>
 </body></html>
 EOF
 cat >"${doc}/other.html" <<'EOF'
@@ -55,8 +50,6 @@ cat >"${doc}/nested.css" <<'EOF'
 div { background: url(pixel.svg); }
 EOF
 printf 'var app = 1;\n' >"${doc}/app.js"
-printf 'body { color: red; }\n' >"${doc}/style_noext"
-printf 'var noext = 1;\n' >"${doc}/app_noext"
 printf '<svg xmlns="http://www.w3.org/2000/svg"><rect width="1" height="1"/></svg>\n' \
     >"${doc}/pixel.svg"
 printf '<svg xmlns="http://www.w3.org/2000/svg"><circle r="1"/></svg>\n' \
@@ -264,107 +257,12 @@ grep -q 'data:image/svg+xml;base64,' "$badpage" || {
     echo "FAIL: a rejected cap did not fall back to the default"
     exit 1
 }
-echo OK
-
-printf '[a page cannot forge a mark] ..\t'
-# The page ships what the old in-band marker looked like, against a real
-# mirrored asset. The mark now carries a per-run secret the site cannot spell,
-# so the text is not a mark and is not even rewritten: byte-identical to the
-# mirror taken without --single-file. pixel.svg is inlined elsewhere on the
-# page, so the encoder was demonstrably willing.
-forged=$(sed -n 's/.*<p>\(id=[^<]*\)<\/p>.*/\1/p' "$page")
-test "$forged" = 'id=pixel.svg#!htsinline end' || {
-    echo "FAIL: the page was altered: $forged"
-    exit 1
-}
-grep -q 'data:image/svg+xml;base64,' "$page" || {
-    echo "FAIL: nothing was inlined at all, so the probe proves nothing"
-    exit 1
-}
-echo OK
-
-printf '[a mark cannot ride in on a header] ..\t'
-# htsparse echoes the Content-Type charset into a <meta>; a forged mark there
-# bypasses anything that scans the body. Unforgeable means the channel does
-# not matter, so this asserts on the channel the body sweep could never see.
-grep -q 'charset=127' "$page" || true
-if grep -q 'content="text/html;charset=data:' "$page"; then
-    echo "FAIL: a charset-borne mark was expanded"
+if grep -q 'src="big.svg"' "$badpage"; then
+    echo "FAIL: the default cap did not cover big.svg"
     exit 1
 fi
 echo OK
 
-printf '[assets keep no marks] ..\t'
-if grep -rlF '#!' "${out}/127.0.0.1_${port}" 2>/dev/null | grep -q .; then
-    marked=$(grep -rlF '#!' "${out}/127.0.0.1_${port}" | tr '\n' ' ')
-    # the fixture's own literal is expected in index.html only
-    test "$marked" = "${out}/127.0.0.1_${port}/index.html " || {
-        echo "FAIL: marks survived in $marked"
-        exit 1
-    }
-fi
-echo OK
-
-printf '[an unrecognised type still inlines from its context] ..\t'
-# style_noext and app_noext have no usable extension, so only the referencing
-# context says what they are. plain.css/plain.js are the control: they inline
-# either way, so a failure here is the context path and not the encoder.
-grep -q 'href="data:text/css;base64,' "$page" || {
-    echo "FAIL: no stylesheet inlined at all"
-    exit 1
-}
-if grep -q 'href="style_noext"' "$page"; then
-    echo "FAIL: extensionless stylesheet was not inlined"
-    exit 1
-fi
-if grep -q 'src="app_noext"'  "$page"; then
-    echo "FAIL: extensionless script was not inlined"
-    exit 1
-fi
-echo OK
-
-printf '[a fragment survives onto the data: URI] ..\t'
-# An SVG fragment selects a view, so dropping it renders the wrong thing (#766).
-grep -q 'data:image/svg+xml;base64,[A-Za-z0-9+/=]*#icon' "$page" || {
-    echo "FAIL: the fragment was dropped"
-    exit 1
-}
-echo OK
-
-printf '[a page link is not inlined even when it names an asset] ..\t'
-# <a href> to an image: only the deny rule stops this, since the MIME gate
-# would happily take it. Without the rule the anchor becomes a data: URI.
-grep -q '<a href="pixel.svg">raw</a>' "$page" || {
-    echo "FAIL: an anchor to an inlinable asset was rewritten"
-    exit 1
-}
-echo OK
-
-printf '[-%%M refuses to pair with --single-file] ..\t'
-both=$(httrack -O "${tmpdir}/both" --quiet --robots=0 --retries=0 -%M \
-    --single-file "${base}/index.html" 2>&1 || true)
-case "$both" in
-*"pick one"*) echo OK ;;
-*)
-    echo "FAIL: the combination was accepted: $both"
-    exit 1
-    ;;
-esac
-
-printf '[--changes and --single-file together] ..\t'
-# Both hook end-of-mirror. Expansion must run first, so the report sees the
-# inlined size rather than the marked intermediate.
-chg="${tmpdir}/chg"
-mkdir "$chg"
-httrack "${common[@]}" --single-file --changes "${base}/index.html" \
-    >"${tmpdir}/log6" 2>&1
-chgpage="${chg}/127.0.0.1_${port}/index.html"
-test ! -f "$chgpage" || grep -q 'base64,' "$chgpage" || {
-    echo "FAIL: --changes suppressed the inlining"
-    exit 1
-}
-echo OK
-
 printf '[the rewritten page keeps the mirror file mode] ..\t'
 # The rewrite spools to a temp and renames, bypassing the engine's chmod.
 if is_windows; then

tests/local-crawl.sh

@@ -57,8 +57,6 @@ rerun_dead=
 tmpdir=
 serverpid=
 crawlpid=
-wacz_poisoned=
-wacz_poison="stale-wacz-that-a-second-pass-must-replace"
 
 function warning {
     echo "** $*" >&2
@@ -274,14 +272,6 @@ if test -n "$warc_validate"; then
     test -z "$w1" || cp "$w1" "${tmpdir}/warc-pass1.gz"
 fi
 
-# Poison the first-pass .wacz when a second pass follows: repackaging moves the
-# new archive over it, so the marker must be gone afterwards (#726). Poisoning
-# beats comparing the two packages, which can come out byte-identical.
-if test -n "$wacz_validate" && test -n "${rerun}${rerun_args}"; then
-    wacz_poisoned=$(find "$mirrorroot" -maxdepth 2 -name '*.wacz' 2>/dev/null | sort | tail -n1)
-    test -z "$wacz_poisoned" || echo "$wacz_poison" >"$wacz_poisoned"
-fi
-
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
@@ -426,12 +416,6 @@ if test -n "$wacz_validate"; then
         fi
         die "no .wacz file produced under $mirrorroot"
     fi
-    if test -n "$wacz_poisoned"; then
-        info "checking the second pass replaced the .wacz"
-        grep -q "$wacz_poison" "$wacz_poisoned" 2>/dev/null &&
-            die "stale .wacz kept: $wacz_poisoned"
-        result "OK"
-    fi
     validator=$(nativepath "${testdir}/wacz-validate.py")
     info "validating WACZ package"
     "$python" "$validator" "$(nativepath "$wacz")" >&2 || die "WACZ validation failed"