htsparse: don't treat XHR.open's method argument as a URL (#218)

The JavaScript URL detector matched `.open(` for window.open("url",...)
and captured the first argument as a link. XMLHttpRequest.open(method,
url) puts the HTTP method first, so `xhr.open("GET", "ajax_info.txt")`
turned "GET" into a bogus link, rewritten to "GET.html" on a live server.

Reject a first argument that is exactly an HTTP method, mirroring the
existing ensure_not_mime guard. window.open(url) is unaffected; the real
XHR url (the second argument) is still picked up by the dirty parser.

Closes #218

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsparse.c

@@ -296,6 +296,27 @@ static const char *html_inline_safe(const char *src, char *dst, size_t size) {
   return dst;
 }
 
+/* Byte before html, or a space sentinel at the buffer start where html[-1]
+   would underflow; space reads as the word boundary the guards want there. */
+static HTS_INLINE char html_prevc(const char *html, const char *start) {
+  return html > start ? html[-1] : ' ';
+}
+
+/* True if [s, s+len) is exactly an HTTP method token (XHR.open's first
+   argument is a method, not a URL: #218). Case-insensitive. */
+static int is_http_method(const char *s, size_t len) {
+  static const char *const methods[] = {"GET",    "POST",  "PUT",
+                                        "DELETE", "HEAD",  "OPTIONS",
+                                        "PATCH",  "TRACE", NULL};
+  int i;
+
+  for (i = 0; methods[i] != NULL; i++) {
+    if (strlen(methods[i]) == len && strfield(s, methods[i]) == (int) len)
+      return 1;
+  }
+  return 0;
+}
+
 /* Main parser */
 int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
   char catbuff[CATBUFF_SIZE];
@@ -556,7 +577,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   if (opt->getmode & HTS_GETMODE_HTML) {
                     p = strfield(html, "title");
                     if (p) {
-                      if (*(html - 1) == '/')
+                      if (html_prevc(html, r->adr) == '/')
                         p = 0;  // /title
                     } else {
                       if (strfield(html, "/html"))
@@ -1341,6 +1362,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     int can_avoid_quotes = 0;
                     char quotes_replacement = '\0';
                     int ensure_not_mime = 0;
+                    // .open(method,url): reject an HTTP-method first arg (#218)
+                    int ensure_not_method = 0;
+                    // @import: the quoted token is the URL; a trailing
+                    // media/supports/layer condition is not part of it
+                    int is_import = 0;
 
                     if (inscript_tag)
                       expected_end = ";\"\'";   // voir a href="javascript:doc.location='foo'"
@@ -1357,9 +1383,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       if (!nc)
                         nc = strfield(html, ":location");        // javascript:location="doc"
                       if (!nc) {        // location="doc"
-                        if ((nc = strfield(html, "location"))
+                        if ((nc = strfield(html, "location")) &&
-                            && !isspace(*(html - 1))
+                            !isspace(html_prevc(html, r->adr)))
-                          )
                           nc = 0;
                       }
                       if (!nc)
@@ -1369,6 +1394,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           expected = '(';       // parenthèse
                           expected_end = "),";  // fin: virgule ou parenthèse
                           ensure_not_mime = 1;  //* ensure the url is not a mime type */
+                          ensure_not_method = 1; // xhr.open: don't grab method
                         }
                       if (!nc)
                         if ((nc = strfield(html, ".replace"))) { // window.replace("url")
@@ -1380,7 +1406,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           expected = '(';       // parenthèse
                           expected_end = ")";   // fin: parenthèse
                         }
-                      if (!nc && (nc = strfield(html, "url")) && (!isalnum(*(html - 1))) && *(html - 1) != '_') {  // url(url)
+                      if (!nc && (nc = strfield(html, "url")) &&
+                          (!isalnum(html_prevc(html, r->adr))) &&
+                          html_prevc(html, r->adr) != '_') { // url(url)
                         expected = '('; // parenthèse
                         expected_end = ")";     // fin: parenthèse
                         can_avoid_quotes = 1;
@@ -1390,6 +1418,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         if ((nc = strfield(html, "import"))) {   // import "url"
                           if (is_space(*(html + nc))) {
                             expected = 0;       // no char expected
+                            is_import = 1;
                           } else
                             nc = 0;
                         }
@@ -1407,6 +1436,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           if ((*a == 34) || (*a == '\'') || (can_avoid_quotes)) {
                             const char *b, *c;
                             int ndelim = 1;
+                            int valid_url = 0;
 
                             if ((*a == 34) || (*a == '\''))
                               a++;
@@ -1421,12 +1451,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                 b++;
                             }
                             c = b--;
-                            c += ndelim;
+                            // no closing delimiter here (truncated input):
-                            while(*c == ' ')
+                            // Don't scan past the buffer NUL or capture it.
-                              c++;
+                            if (*c != '\0') {
-                            if ((strchr(expected_end, *c)) || (*c == '\n')
+                              c += ndelim;
-                                || (*c == '\r')) {
+                              while (*c == ' ')
-                              c -= (ndelim + 1);
+                                c++;
+                              valid_url =
+                                  (strchr(expected_end, *c)) || (*c == '\n') ||
+                                  (*c == '\r') ||
+                                  (is_import && *(b + 1 + ndelim) == ' ');
+                            }
+                            if (valid_url) {
+                              // URL end = last char (b), not the delimiter
+                              c = b;
                               if ((int) (c - a + 1)) {
                                 if (ensure_not_mime) {
                                   int i = 0;
@@ -1442,6 +1480,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                     i++;
                                   }
                                 }
+                                // XHR.open's "GET" etc. is a method, not a URL
+                                if (a != NULL && ensure_not_method &&
+                                    is_http_method(a, (size_t) (c - a + 1))) {
+                                  a = NULL;
+                                }
                                 // Check for bogus links (Vasiliy)
                                 if (a != NULL) {
                                   const size_t size = c - a + 1;
@@ -1485,7 +1528,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                 }
                               }
                             }
-
                           }
                         }
                       }
@@ -1692,6 +1734,24 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                                               hts_nodetect[i -
                                                                            1]);
                                               }
+                                              // xmlns / xmlns:prefix declare
+                                              // XML namespaces, not resources
+                                              // (#191)
+                                              else {
+                                                const int xl = strfield(
+                                                    intag_startattr, "xmlns");
+                                                const char xc =
+                                                    intag_startattr[xl];
+                                                if (xl &&
+                                                    (xc == ':' || xc == '=' ||
+                                                     is_space(xc))) {
+                                                  url_ok = 0;
+                                                  hts_log_print(
+                                                      opt, LOG_DEBUG,
+                                                      "dirty parsing: xmlns "
+                                                      "namespace avoided");
+                                                }
+                                              }
                                             }
                                   }
 

tests/01_engine-parse.test

@@ -154,4 +154,114 @@ grep -Eq "style=\"background-image:url\('ibgs\.gif'\)\"" "$saved2" ||
 grep -q 'title="file://' "$saved2" ||
     ! echo "FAIL: a no-detect attribute (title) was wrongly rewritten" || exit 1
 
+# xmlns / xmlns:prefix decls must not be crawled (#191). Local file:// targets so a
+# regression downloads them; each is the LAST attr (heuristic only scans a value before '>').
+site3="$tmp/xmlns"
+mkdir -p "$site3"
+for f in ns og rdfs real; do gif "$site3/$f.gif"; done
+cat >"$site3/index.html" <<EOF
+<html xmlns="file://$site3/ns.gif"><body>
+<svg xmlns:og="file://$site3/og.gif"></svg>
+<div class="c" xmlns:rdfs="file://$site3/rdfs.gif"></div>
+<a href="file://$site3/real.gif">real link</a>
+</body></html>
+EOF
+out3="$tmp/xmlns-out"
+crawl "$site3/index.html" "$out3"
+
+# the real link is still captured
+found "real.gif" "$out3"
+# namespace-declaration targets must not be fetched (default + prefixed forms)
+notfound "ns.gif" "$out3"
+notfound "og.gif" "$out3"
+notfound "rdfs.gif" "$out3"
+
+# CSS @import (#94): every form's target is captured, crawling the .css directly.
+# The "cond"/"sup"/"spc" cases carry a trailing media/supports/layer condition (or
+# a space before ';'); they are the negative controls: without the parser fix the
+# URL is dropped, so a regression fails these found() checks.
+site4="$tmp/cssimport"
+mkdir -p "$site4"
+for f in nq dqu squ dqs sqs med cond sup lay spc; do printf 'body{}\n' >"$site4/$f.css"; done
+cat >"$site4/main.css" <<'EOF'
+@import url(nq.css);
+@import url("dqu.css");
+@import url('squ.css');
+@import "dqs.css";
+@import 'sqs.css';
+@import url(med.css) screen and (min-width: 400px);
+@import "cond.css" screen;
+@import "sup.css" supports(display: flex);
+@import url(lay.css) layer(base);
+@import "spc.css" ;
+EOF
+out4="$tmp/cssimport-out"
+crawl "$site4/main.css" "$out4"
+for f in nq dqu squ dqs sqs med cond sup lay spc; do found "$f.css" "$out4"; done
+
+# Over-capture guard: the trailing condition is not part of the URL, so it must
+# survive the rewrite verbatim. A regression that grabs it would mangle these.
+m4=$(find "$out4" -type f -path '*/file/*' -name main.css -print -quit)
+test -n "$m4" || ! echo "FAIL: saved main.css not found" || exit 1
+for cond in '@import "cond.css" screen;' 'supports(display: flex)' 'layer(base)'; do
+    grep -Fq "$cond" "$m4" ||
+        ! echo "FAIL #94: '$cond' altered on rewrite (condition captured as URL?)" || exit 1
+done
+
+# Malformed input: an unterminated @import quote (truncated CSS) must not crash or
+# capture a bogus link; a valid sibling import is still captured. Guards a heap
+# overflow on the URL-end scan that aborts under ASan (CI sanitizer job).
+site5="$tmp/cssimport-trunc"
+mkdir -p "$site5"
+printf 'body{}\n' >"$site5/good.css"
+printf '@import "good.css";\n@import "trunc' >"$site5/main.css"
+out5="$tmp/cssimport-trunc-out"
+crawl "$site5/main.css" "$out5"
+found "good.css" "$out5"
+notfound "trunc" "$out5"
+
+# Offset-0 underflow (#396): a token at the buffer start makes the detector's
+# word-boundary guard read *(html-1) one byte early (aborts under ASan). The
+# url() target is still captured; here it just must not underflow.
+site6="$tmp/parse-off0"
+mkdir -p "$site6"
+printf 'body{}\n' >"$site6/off0.css"
+printf 'url(off0.css)\n' >"$site6/main.css"
+out6="$tmp/parse-off0-out"
+crawl "$site6/main.css" "$out6"
+found "off0.css" "$out6"
+
+# XMLHttpRequest.open(method, url) (#218): the first argument is an HTTP method,
+# not a URL. Without the fix "GET" is captured as a link and fetched (the offline
+# fixture saves a bare file named GET; a live server mangles it to GET.html).
+# window.open(url) detection must be unaffected.
+site7="$tmp/xhropen"
+mkdir -p "$site7"
+gif "$site7/winopen.gif"
+cat >"$site7/index.html" <<EOF
+<html><body><script>
+var x = new XMLHttpRequest();
+x.open("GET", "ajax_info.txt");
+var y = new XMLHttpRequest();
+y.open("Post", "submit.cgi");
+window.open("file://$site7/winopen.gif");
+</script></body></html>
+EOF
+out7="$tmp/xhropen-out"
+crawl "$site7/index.html" "$out7"
+# negative control: without the fix a file named exactly GET is downloaded
+notfound "GET" "$out7"
+# methods are matched case-insensitively (XHR spec normalizes them): a mixed-case
+# method is rejected too, so a file named Post must not appear either
+notfound "Post" "$out7"
+# regression guard: window.open(url) is still detected, so its absolute URL is
+# rewritten to a local link. The rewrite only happens if the parser saw it, so
+# these two assertions fail if .open detection broke (not a trivial --near save).
+saved7=$(savedhtml "$out7")
+test -n "$saved7" || ! echo "FAIL: saved xhr page not found" || exit 1
+grep -Fq 'window.open("winopen.gif")' "$saved7" ||
+    ! echo "FAIL #218: window.open(url) no longer detected/rewritten" || exit 1
+! grep -Fq 'window.open("file://' "$saved7" ||
+    ! echo "FAIL #218: window.open URL left absolute (not rewritten)" || exit 1
+
 exit 0