The wizard-precedence comments restate the code (#1272)

Review of #1257 flagged seven comment blocks that restate the code or narrate the change rather than saying why. This trims them to the house one-line default: a five-line re-listing of httrack.c's query3 answer codes, an aside about where the scenarios abort, wording that called the counter a block size, and a six-line header on `hts_wizard_insert_filters`.

The contract facts stay. The log-echo caller reads back the range the function reports, so the return count and the placement of the inserted filters at the tail of the block are still documented at the declaration.

Comment-only: a whitespace-ignoring diff shows no code change.

Signed-off-by: Xavier Roche <xroche@gmail.com>
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/htscore.c

@@ -608,9 +608,7 @@ int httpmirror(char *url1, httrackp * opt) {
     XH_extuninit;
     return 0;
   }
-  opt->filters.filters = &filters;
-  //
-  opt->filters.filptr = &filptr;
+  filters_bind(opt, &filters, &filptr);
 
   // hash table
   opt->hash = &hash;
@@ -2426,6 +2424,12 @@ void host_ban(httrackp * opt, int ptr,
   }
 }
 
+void filters_bind(httrackp *opt, char ***ptrfilters, int *filptr) {
+  opt->filters.filters = ptrfilters;
+  opt->filters.filptr = filptr;
+  opt->wizard_filters = 0;
+}
+
 int filters_init(char ***ptrfilters, int maxfilter, int filterinc) {
   char **filters = *ptrfilters;
   int filter_max = maximum(maxfilter, 128);

src/htscore.h

@@ -387,6 +387,10 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
 
 int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
 
+/* Binds this crawl's filter array to `opt`, and empties the wizard block with
+   it: opt->wizard_filters indexes into that array and outlives no crawl. */
+void filters_bind(httrackp *opt, char ***ptrfilters, int *filptr);
+
 int fspc(httrackp * opt, FILE * fp, const char *type);
 
 char *next_token(char *p, int flag);

src/htsopt.h

@@ -568,9 +568,12 @@ struct httrackp {
      growing it would shift every httrackp field declared after it. */
   void *sitemap_state; /**< hts_sitemap_state*, or NULL. Tail: ABI */
   void *singlefile_state; /**< hts_singlefile_state*, or NULL. Tail: ABI */
-  String host_alias; /**< --host-alias: '\n'-separated "alias[,alias...]=host"
-                          rules folding the hostnames of one site onto a single
-                          canonical host. Tail: ABI */
+  String host_alias;  /**< --host-alias: '\n'-separated "alias[,alias...]=host"
+                           rules folding the hostnames of one site onto a single
+                           canonical host. Tail: ABI */
+  int wizard_filters; /**< count of filters the wizard has inserted, held at the
+                           low indices of the array. Live state, so copy_htsopt
+                           must leave it alone. Tail: ABI */
 };
 
 /* Running statistics for a mirror. */

src/htsselftest.c

@@ -4454,6 +4454,176 @@ static int st_wizardverdict(httrackp *opt, int argc, char **argv) {
 
 #undef PRIO_UNSET
 
+/* Resets the wizard's filter array to the command-line filters `cmd`
+   (NULL-terminated). */
+static void wz_seed(httrackp *opt, char **filters, int *filptr,
+                    const char *const *cmd) {
+  int i;
+
+  *filptr = 0;
+  opt->wizard_filters = 0;
+  for (i = 0; cmd != NULL && cmd[i] != NULL; i++)
+    strlcpybuff(filters[(*filptr)++], cmd[i], HTS_FILTER_SLOT_SIZE);
+}
+
+/* Asserts the array holds exactly `want`, in order, naming the slot that
+   differs. */
+static void wz_holds(char **filters, int filptr, const char *const *want) {
+  int i;
+
+  for (i = 0; want[i] != NULL; i++) {
+    if (i >= filptr || strcmp(filters[i], want[i]) != 0)
+      fprintf(stderr, "filter %d: got [%s], want [%s]\n", i,
+              i < filptr ? filters[i] : "<past the end>", want[i]);
+    assertf(i < filptr);
+    assertf(strcmp(filters[i], want[i]) == 0);
+  }
+  if (filptr != i)
+    fprintf(stderr, "filter %d: got [%s], want nothing\n", i,
+            i < filptr ? filters[i] : "<past the end>");
+  assertf(filptr == i);
+}
+
+/* Drives hts_wizard_insert_filters(): prints the array the given answers build
+   over the command-line filters, or asserts the precedence rules. */
+static int st_wizardinsert(httrackp *opt, int argc, char **argv) {
+  const htsfilters saved = opt->filters;
+  const int savedwizard = opt->wizard_filters;
+  const int savedmax = opt->maxfilter;
+  char **filters = NULL;
+  int filptr = 0;
+  int i;
+
+  assertf(filters_init(&filters, opt->maxfilter, 0) != 0);
+  opt->filters.filters = &filters;
+  opt->filters.filptr = &filptr;
+  opt->wizard_filters = 0;
+  /* Answers are httrack.c's query3 codes; seeker_up is pinned off, so answer 5
+     takes its directory branch (tests/264 covers the host one). */
+#define INSERT(n, adr, fil)                                                    \
+  hts_wizard_insert_filters(opt, (n), (adr), (fil), HTS_FALSE)
+#define HOLDS(...)                                                             \
+  do {                                                                         \
+    const char *const want[] = {__VA_ARGS__, NULL};                            \
+    wz_holds(filters, filptr, want);                                           \
+  } while (0)
+#define SEED(...)                                                              \
+  do {                                                                         \
+    const char *const cmd[] = {__VA_ARGS__, NULL};                             \
+    wz_seed(opt, filters, &filptr, cmd);                                       \
+  } while (0)
+#define RESET() wz_seed(opt, filters, &filptr, NULL)
+/* what the array as it stands decides for `url` */
+#define VERDICT(url) fa_strjoker(0, filters, filptr, (url), NULL, NULL, NULL)
+
+  if (argc >= 2) {
+    int sep = argc;
+
+    for (i = 2; i < argc; i++) {
+      if (strcmp(argv[i], "@") == 0) {
+        sep = i;
+        break;
+      }
+    }
+    for (i = sep + 1;
+         i < argc && filptr + HTS_WIZARD_MAX_FILTERS < opt->maxfilter; i++)
+      strlcpybuff(filters[filptr++], argv[i], HTS_FILTER_SLOT_SIZE);
+    for (i = 2; i < sep; i++)
+      INSERT(atoi(argv[i]), argv[0], argv[1]);
+    for (i = 0; i < filptr; i++)
+      printf("%s%s", i != 0 ? " " : "", filters[i]);
+    printf("\n");
+    goto done;
+  }
+
+  /* wizard_filters indexes one crawl's array, so binding another resets it */
+  {
+    char **other = NULL;
+    int otherptr = 7;
+
+    opt->wizard_filters = 3;
+    filters_bind(opt, &other, &otherptr);
+    assertf(opt->wizard_filters == 0);
+    assertf(opt->filters.filters == &other && opt->filters.filptr == &otherptr);
+    filters_bind(opt, &filters, &filptr);
+  }
+
+  /* a counter that outlived its array still cannot index past the end */
+  SEED("-*.zip");
+  opt->wizard_filters = 99;
+  INSERT(6, "h", "/dir/two.html");
+  assertf(opt->wizard_filters <= filptr);
+
+  /* the correction case: "ignore this link", then "mirror the whole host" */
+  RESET();
+  INSERT(0, "h", "/dir/one.html");
+  HOLDS("-h/dir/one.html");
+  assertf(opt->wizard_filters == 1);
+  assertf(VERDICT("h/dir/one.html") == -1);
+  INSERT(6, "h", "/dir/two.html");
+  HOLDS("-h/dir/one.html", "+h/*");
+  assertf(opt->wizard_filters == 2);
+  assertf(VERDICT("h/dir/one.html") == 1);
+
+  /* reversing the answers reverses the outcome, or the block is not ordered */
+  RESET();
+  INSERT(6, "h", "/dir/two.html");
+  INSERT(0, "h", "/dir/one.html");
+  HOLDS("+h/*", "-h/dir/one.html");
+  assertf(VERDICT("h/dir/one.html") == -1);
+  assertf(VERDICT("h/dir/two.html") == 1);
+
+  /* no answer reaches above the command line */
+  SEED("-h/*.zip", "+h/keep/*");
+  INSERT(6, "h", "/dir/two.html");
+  HOLDS("+h/*", "-h/*.zip", "+h/keep/*");
+  assertf(opt->wizard_filters == 1);
+  assertf(VERDICT("h/a.zip") == -1);
+  assertf(VERDICT("h/dir/two.html") == 1);
+
+  /* the primary link records its scope first, so a later answer overrides it */
+  RESET();
+  INSERT(7, "h", "/dir/index.html");
+  INSERT(0, "h", "/dir/one.html");
+  HOLDS("+h/dir/*[file]", "-h/dir/one.html");
+  assertf(VERDICT("h/dir/one.html") == -1);
+
+  /* both halves of a host-scope answer land in the block, in emission order */
+  SEED("-*.zip");
+  INSERT(HTS_WIZARD_SCOPE_INCLUDE + 1, "www.example.co.uk", "/x.html");
+  HOLDS("+*.example.co.uk/*", "+example.co.uk/*", "-*.zip");
+  assertf(opt->wizard_filters == 2);
+
+  /* an answer that emits nothing leaves the block and the array unchanged */
+  SEED("-*.zip");
+  INSERT(4, "h", "/dir/one.html");
+  INSERT(50, "h", "/dir/one.html");
+  INSERT(3, "h", "/dir/one.html");
+  HOLDS("-*.zip");
+  assertf(opt->wizard_filters == 0);
+
+  /* answers 1, 5 and 7 emit nothing when the file part has no directory */
+  RESET();
+  INSERT(1, "h", "one.html");
+  INSERT(5, "h", "one.html");
+  INSERT(7, "h", "one.html");
+  assertf(filptr == 0 && opt->wizard_filters == 0);
+
+  printf("wizardinsert self-test OK\n");
+done:
+#undef INSERT
+#undef HOLDS
+#undef SEED
+#undef RESET
+#undef VERDICT
+  freet(filters[0]);
+  freet(filters);
+  opt->filters = saved;
+  opt->wizard_filters = savedwizard;
+  opt->maxfilter = savedmax;
+  return 0;
+}
+
 /* #159: hts_redirect_same_savefile decides whether a redirect is a same-file
  * alias. */
 static int st_redirect_samefile(httrackp *opt, int argc, char **argv) {
@@ -9973,6 +10143,8 @@ static const struct selftest_entry {
      st_wizardscopeanswer},
     {"wizardverdict", "[<answer>]", "what a wizard answer applies",
      st_wizardverdict},
+    {"wizardinsert", "[<adr> <fil> [answer...] [@ filter...]]",
+     "where a wizard answer lands in the filter array", st_wizardinsert},
     {"mime", "<filename>", "MIME type for a filename", st_mime},
     {"charset", "<charset> <hex:..|string>",
      "convert a string to UTF-8 from a charset", st_charset},

src/htswizard.c

@@ -42,25 +42,6 @@ Please visit our Website: http://www.httrack.com
 #include <ctype.h>
 /* END specific definitions */
 
-// libérer filters[0] pour insérer un élément dans filters[0]
-/* Per-slot capacity of the filters array, matching the slot stride allocated by
-   filters_init() in htscore.c (HTS_URLMAXSIZE * 2). */
-#define HTS_FILTER_SLOT_SIZE (HTS_URLMAXSIZE * 2)
-
-#define HT_INSERT_FILTERS0                                                     \
-  do {                                                                         \
-    int i;                                                                     \
-    if (*opt->filters.filptr > 0) {                                            \
-      for (i = (*opt->filters.filptr) - 1; i >= 0; i--) {                      \
-        strlcpybuff((*opt->filters.filters)[i + 1],                            \
-                    (*opt->filters.filters)[i], HTS_FILTER_SLOT_SIZE);         \
-      }                                                                        \
-    }                                                                          \
-    (*opt->filters.filters)[0][0] = '\0';                                      \
-    (*opt->filters.filptr)++;                                                  \
-    assertf((*opt->filters.filptr) < opt->maxfilter);                          \
-  } while (0)
-
 /* "embedded" */
 htspair_t hts_detect_embed[] = {
   {"img", "src"},
@@ -322,6 +303,55 @@ void hts_wizard_answer_filter(htsbuff *f, int slot, int n, const char *adr,
   }
 }
 
+/* Inserts `pattern` at index `pos`, shifting the filters from there up a slot.
+   The caller has already ensured the array has room. */
+static void filters_insert_at(httrackp *opt, int pos, const char *pattern) {
+  char **const filters = *opt->filters.filters;
+  int i;
+
+  assertf(pos >= 0 && pos <= *opt->filters.filptr);
+  for (i = *opt->filters.filptr; i > pos; i--)
+    strlcpybuff(filters[i], filters[i - 1], HTS_FILTER_SLOT_SIZE);
+  strlcpybuff(filters[pos], pattern, HTS_FILTER_SLOT_SIZE);
+  (*opt->filters.filptr)++;
+  assertf((*opt->filters.filptr) < opt->maxfilter);
+}
+
+int hts_wizard_insert_filters(httrackp *opt, int n, const char *adr,
+                              const char *fil, hts_boolean seeker_up) {
+  char BIGSTK pattern[HTS_FILTER_SLOT_SIZE];
+  htsbuff f = htsbuff_array(pattern);
+  int slot;
+
+  /* grow first: a host-scope answer emits two filters */
+  if ((*opt->filters.filptr) + 2 >= opt->maxfilter) {
+    opt->maxfilter += HTS_FILTERSINC;
+    if (filters_init(opt->filters.filters, opt->maxfilter, HTS_FILTERSINC) ==
+        0) {
+      printf("PANIC! : Too many filters : >%d [%d]\n", *opt->filters.filptr,
+             __LINE__);
+      fflush(stdout);
+      hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)",
+                    *opt->filters.filptr);
+      hts_log_print(
+          opt, LOG_INFO,
+          "To avoid that: use #F option for more filters (example: -#F5000)");
+      assertf("too many filters - giving up" == NULL); // wild..
+    }
+  }
+  /* a counter outliving its array (an opt reused for a second crawl) would
+     index past the end */
+  if (opt->wizard_filters > *opt->filters.filptr)
+    opt->wizard_filters = *opt->filters.filptr;
+  for (slot = 0; slot < HTS_WIZARD_MAX_FILTERS; slot++) {
+    hts_wizard_answer_filter(&f, slot, n, adr, fil, seeker_up);
+    if (f.len == 0)
+      break;
+    filters_insert_at(opt, opt->wizard_filters++, pattern);
+  }
+  return slot;
+}
+
 void hts_wizard_apply_verdict(httrackp *opt, int n, const char *adr,
                               const char *fil, int *forbidden_url,
                               int *set_prio_to) {
@@ -894,48 +924,25 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
           n = force_mirror;
       }
 
-      /* sanity check - reallocate filters HERE (a host-scope answer emits two)
-       */
-      if ((*_FILTERS_PTR) + 2 >= opt->maxfilter) {
-        opt->maxfilter += HTS_FILTERSINC;
-        if (filters_init(&_FILTERS, opt->maxfilter, HTS_FILTERSINC) == 0) {
-          printf("PANIC! : Too many filters : >%d [%d]\n", (*_FILTERS_PTR),
-                 __LINE__);
-          fflush(stdout);
-          hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)",
-                        (*_FILTERS_PTR));
-          hts_log_print(opt, LOG_INFO,
-                        "To avoid that: use #F option for more filters (example: -#F5000)");
-          assertf("too many filters - giving up" == NULL);      // wild..
-        }
-      }
-      // here we have enough room for a new filter if necessary
-
       hts_wizard_apply_verdict(opt, n, adr, fil, &forbidden_url, set_prio_to);
-
-      /* the pattern half of the answer */
       {
-        char BIGSTK pattern[HTS_FILTER_SLOT_SIZE];
-        /* the log echoes the inserted slots, so it cannot drift from them */
-        char BIGSTK list[HTS_WIZARD_MAX_FILTERS * (HTS_FILTER_SLOT_SIZE + 1)];
-        htsbuff f = htsbuff_array(pattern);
-        htsbuff added = htsbuff_array(list);
-        int slot;
+        const int inserted = hts_wizard_insert_filters(
+            opt, n, adr, fil,
+            (opt->seeker & HTS_SEEKER_UP) != 0 ? HTS_TRUE : HTS_FALSE);
 
-        for (slot = 0; slot < HTS_WIZARD_MAX_FILTERS; slot++) {
-          hts_wizard_answer_filter(
-              &f, slot, n, adr, fil,
-              (opt->seeker & HTS_SEEKER_UP) != 0 ? HTS_TRUE : HTS_FALSE);
-          if (f.len == 0)
-            break;
-          HT_INSERT_FILTERS0; // insert at slot 0
-          strlcpybuff(_FILTERS[0], pattern, HTS_FILTER_SLOT_SIZE);
-          if (added.len != 0)
-            htsbuff_cat(&added, " ");
-          htsbuff_cat(&added, _FILTERS[0]);
-        }
         /* the built-in query3 answers "" for nobody, so ask who replied */
         if (s != NULL && HAS_CALLBACK(opt, query3)) {
+          char BIGSTK list[HTS_WIZARD_MAX_FILTERS * (HTS_FILTER_SLOT_SIZE + 1)];
+          htsbuff added = htsbuff_array(list);
+          int slot;
+
+          /* read the slots back, so the log cannot drift from them */
+          for (slot = opt->wizard_filters - inserted;
+               slot < opt->wizard_filters; slot++) {
+            if (added.len != 0)
+              htsbuff_cat(&added, " ");
+            htsbuff_cat(&added, _FILTERS[slot]);
+          }
           hts_log_print(
               opt, LOG_NOTICE, "(wizard) answer '%s' (n=%d) for %s%s: %s%s%s",
               s, n, adr, fil,
@@ -1089,5 +1096,3 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
   }
   return jok;
 }
-
-#undef HT_INSERT_FILTERS0

src/htswizard.h

@@ -57,6 +57,10 @@ hts_boolean hts_robots_forbids(httrackp *opt, const char *adr, const char *fil,
                                hts_boolean filters_decided,
                                hts_boolean filters_refused);
 
+/* Per-slot capacity of the filters array, matching the slot stride allocated by
+   filters_init() in htscore.c (HTS_URLMAXSIZE * 2). */
+#define HTS_FILTER_SLOT_SIZE (HTS_URLMAXSIZE * 2)
+
 /* Most filters one wizard answer can add. Slots must stay contiguous: the
    caller stops at the first empty one. */
 #define HTS_WIZARD_MAX_FILTERS 2
@@ -68,6 +72,14 @@ hts_boolean hts_robots_forbids(httrackp *opt, const char *adr, const char *fil,
 void hts_wizard_answer_filter(htsbuff *f, int slot, int n, const char *adr,
                               const char *fil, hts_boolean seeker_up);
 
+/* Records the filters answer `n` leaves behind for the link (adr,fil), on top
+   of the wizard's block at the low indices of opt->filters: last match wins, so
+   a later answer outranks an earlier one and the command-line filters above the
+   block outrank every answer. Returns the number inserted, which sit at the
+   tail of the block, ending at opt->wizard_filters. */
+int hts_wizard_insert_filters(httrackp *opt, int n, const char *adr,
+                              const char *fil, hts_boolean seeker_up);
+
 /* Which host-scope range answer `n` falls in: HTS_TRUE excludes the scope,
    HTS_FALSE includes it, HTS_DEFAULT for any answer outside both ranges. */
 hts_tristate hts_wizard_scope_answer(int n);

tests/258_crawllib.test

@@ -206,6 +206,35 @@ local_crawl --log "${tmpdir}/crawl.log" -O /dev/null "$url"
 assert_eq "crawl-said-this" "$(cat "${tmpdir}/crawl.log")" "--log truncates per crawl"
 ok "--log takes the crawl output, one crawl at a time"
 
+## --stdin feeds the engine, which a redirect on the call cannot be trusted to do
+# Three outcomes on one line, so a closed descriptor is told apart from the empty
+# read /dev/null gives.
+# Written straight out, never through $(cat): with fd 0 closed a command
+# substitution takes the free slot 0 for its pipe and the read blocks forever.
+cat >"${shim}/httrack" <<'SH'
+#!/bin/bash
+printf 'stdin=['
+# uutils cat does not fail on a closed fd, so ask the shell; the subshell keeps
+# an exec redirection error from taking bash 3.2 down with it
+if (exec 3<&0) 2>/dev/null; then cat 2>/dev/null; else printf closed; fi
+printf ']\n'
+SH
+chmod +x "${shim}/httrack"
+
+printf answer >"${tmpdir}/answers"
+# Against a call redirected elsewhere: --stdin is on the engine, so it wins.
+local_crawl --stdin "${tmpdir}/answers" --log "${tmpdir}/crawl.log" \
+    -O /dev/null "$url" </dev/null
+assert_eq "stdin=[answer]" "$(cat "${tmpdir}/crawl.log")" "--stdin FILE"
+local_crawl --stdin closed --log "${tmpdir}/crawl.log" \
+    -O /dev/null "$url" <"${tmpdir}/answers"
+assert_eq "stdin=[closed]" "$(cat "${tmpdir}/crawl.log")" "--stdin closed"
+# Without the option the engine still has a descriptor, whatever the platform
+# gives a background job.
+local_crawl --log "${tmpdir}/crawl.log" -O /dev/null "$url" </dev/null
+assert_eq "stdin=[]" "$(cat "${tmpdir}/crawl.log")" "the default stdin"
+ok "--stdin hands the engine a file, or no descriptor at all"
+
 ## The watchdog reds a crawl that never ends, from a subject: it ends the test
 cat >"${shim}/httrack" <<'SH'
 #!/bin/bash

tests/294_local-wizard-eof.test

@@ -5,8 +5,6 @@ set -eu
 # shellcheck source=tests/crawllib.sh
 . "$(dirname "$0")/crawllib.sh"
 
-httrack=$(command -v httrack) || ! echo "could not find httrack" >&2 || exit 1
-
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_1247.XXXXXX") || exit 1
 cleanup() {
     rm -rf "$tmpdir"
@@ -33,15 +31,14 @@ cat >"${tmpdir}/root/wizardeof/index.html" <<EOF
 EOF
 local_server_start --root "${tmpdir}/root"
 
-# crawl LABEL; the caller redirects the stdin the wizard will read from
+# crawl LABEL STDIN; STDIN is what the wizard reads from
 crawl() {
     local label=$1 rc=0 asked
 
-    run_with_timeout 60 "$httrack" -O "${tmpdir}/$label" -W --robots=0 \
-        --retries=0 "http://127.0.0.1:${SRV_PORT}/wizardeof/index.html" \
-        >"${tmpdir}/$label.log" 2>&1 || rc=$?
+    local_crawl --stdin "$2" --log "${tmpdir}/$label.log" -- \
+        -O "${tmpdir}/$label" -W --robots=0 --retries=0 \
+        "http://127.0.0.1:${SRV_PORT}/wizardeof/index.html" || rc=$?
 
-    test "$rc" -ne 124 || fail "$label: the wizard never returned"
     test "$rc" -eq 0 || fail "$label: the crawl exited $rc"
     # asked once and then stopped: refusing this one link would ask again
     asked=$(grep -c "beyond this mirror scope" "${tmpdir}/$label.log" || true)
@@ -52,6 +49,6 @@ crawl() {
         fail "$label: the refused link was mirrored"
 }
 
-crawl eof </dev/null
+crawl eof /dev/null
 # a closed descriptor sets only ferror, where EOF sets only feof
-crawl closed <&-
+crawl closed closed

tests/296_local-wizard-delayed.test

@@ -8,8 +8,6 @@ set -euo pipefail
 # shellcheck source=tests/crawllib.sh
 . "$(dirname "$0")/crawllib.sh"
 
-httrack=$(httrack_path)
-
 # "mirror this link"
 WIZARD_MIRROR=5
 # spare answers: an empty one means "ignore this link", and EOF spins the
@@ -41,16 +39,10 @@ local_server_start --root "${tmpdir}/root"
 answers="${tmpdir}/answers"
 for _ in $(seq "$WIZARD_SPARES"); do echo "$WIZARD_MIRROR"; done >"$answers"
 
-# redirect here, not in run_with_timeout's args: it backgrounds the job with
-# stdin closed (#1258)
-crawl() { "$httrack" "$@" <"$answers"; }
-
 rc=0
-run_with_timeout "$(crawl_deadline)" crawl --max-time="$CRAWL_MAX_TIME" \
+local_crawl --stdin "$answers" --log "${tmpdir}/crawl.log" -- \
     -O "${tmpdir}/mirror" -W --robots=0 --retries=0 \
-    "http://127.0.0.1:${SRV_PORT}/wizdelayed/index.html" \
-    >"${tmpdir}/crawl.log" 2>&1 || rc=$?
-test "$rc" -ne 124 || fail "the crawl never returned"
+    "http://127.0.0.1:${SRV_PORT}/wizdelayed/index.html" || rc=$?
 test "$rc" -eq 0 || fail "the crawl exited $rc"
 
 grep -q 'beyond this mirror scope' "${tmpdir}/crawl.log" ||

tests/297_engine-wizard-precedence.test

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# #1250: last match wins, so the order below is each answer's precedence.
+
+set -euo pipefail
+
+# shellcheck source=tests/testlib.sh
+. "$(dirname "$0")/testlib.sh"
+
+# answering "ignore this link" and then "mirror the whole host" takes it back
+list=$(httrack -O /dev/null -#test=wizardinsert h /dir/one.html 0 6)
+test "$list" = "-h/dir/one.html +h/*" || fail "answer order: got [$list]"
+# shellcheck disable=SC2086 # the array is one filter per word, by construction
+assert_selftest "verdict=allowed rule=1" filterdual http://h/dir/one.html h/dir/one.html $list
+
+# the reverse order still refuses, so the block is ordered rather than sorted
+list=$(httrack -O /dev/null -#test=wizardinsert h /dir/one.html 6 0)
+test "$list" = "+h/* -h/dir/one.html" || fail "reverse order: got [$list]"
+# shellcheck disable=SC2086
+assert_selftest "verdict=forbidden rule=1" filterdual http://h/dir/one.html h/dir/one.html $list
+
+# a standing command-line rule is not revoked by "mirror the whole host"
+list=$(httrack -O /dev/null -#test=wizardinsert h /dir/one.html 6 @ "-h/*.zip")
+test "$list" = "+h/* -h/*.zip" || fail "command line: got [$list]"
+# shellcheck disable=SC2086
+assert_selftest "verdict=forbidden rule=1" filterdual http://h/a.zip h/a.zip $list
+
+assert_selftest "wizardinsert self-test OK" wizardinsert

tests/crawllib.sh

@@ -104,10 +104,14 @@ local_server_start() {
 # backstops most of these tests lacked: a --max-time cap (skipped when the caller
 # sets its own) and a watchdog above it, so a wedge outliving the engine limit
 # reds the test instead of the 45-minute CI timeout.
-# One option, ahead of any httrack argument; -- ends it:
+# Options, ahead of any httrack argument; -- ends them:
 #   --log FILE     crawl output (default: discarded)
+#   --stdin FILE   what the engine reads on stdin, for a test answering an
+#                  interactive prompt; the word "closed" hands it no descriptor
+#                  at all. Redirecting the local_crawl call instead does not
+#                  reach the engine (#1258).
 local_crawl() {
-    local deadline log=/dev/null arg rc=0
+    local deadline log=/dev/null stdin=() arg rc=0
     deadline=$(crawl_deadline)
     while test $# -gt 0; do
         case $1 in
@@ -115,6 +119,10 @@ local_crawl() {
             log=$2
             shift 2
             ;;
+        --stdin)
+            stdin=(--stdin "$2")
+            shift 2
+            ;;
         --)
             shift
             break
@@ -129,7 +137,8 @@ local_crawl() {
     done
     args+=("$@")
 
-    run_with_timeout "$deadline" httrack "${args[@]}" >"$log" 2>&1 || rc=$?
+    run_with_timeout ${stdin[@]+"${stdin[@]}"} "$deadline" httrack "${args[@]}" \
+        >"$log" 2>&1 || rc=$?
     test "$rc" -ne 124 || fail "crawl watchdog fired after ${deadline}s"
     return "$rc"
 }

tests/testlib.sh

@@ -625,13 +625,28 @@ reap_bounded() {
 # macOS and its signals can't reap httrack.exe on Windows. We poll and kill_tree.
 # All three deadlines below compare strictly: $SECONDS is floored, so a reading of
 # the budget can be a fraction under it, and firing early kills healthy work.
+# One option, ahead of the deadline:
+#   --stdin FILE   read FILE, or nothing at all with the word "closed". A
+#                  redirect on the caller does not reach here: where job control
+#                  is off, bash gives a background job /dev/null (#1258). The
+#                  redirect stays on the job, so the target is still our direct
+#                  child and kill_tree keeps signalling it rather than a wrapper.
 run_with_timeout() {
+    local stdin=''
+    if test "${1:-}" = --stdin; then
+        stdin=$2
+        shift 2
+    fi
     local secs=$1
     shift
     local had_m=
     case "$-" in *m*) had_m=1 ;; esac
     is_windows || set -m # own process group, so kill_tree can signal the group
-    "$@" &
+    case $stdin in
+    '') "$@" & ;;
+    closed) "$@" <&- & ;;
+    *) "$@" <"$stdin" & ;;
+    esac
     local pid=$!
     test -n "$had_m" || is_windows || set +m
     # Read while the job is certainly alive: by kill time /proc/<pid>/winpid is gone.