tests: international crawl tracked the double-encoded behavior

The default.html sub-crawl asserted the pre-fix output: two mojibake
café*.html filenames from double-encoded fetches and 4 errors where
the fix leaves 2 (the caf%e9.html Latin-1 escapes the server really
lacks). Update to the corrected names and counts; the other three
sub-crawls are unchanged. Only runs where ut.httrack.com is reachable
(in-tree builds), which is why just the deb CI job caught it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

fuzz/corpus/meta/meta-content-first.html

@@ -0,0 +1 @@
+<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-truncated.html

@@ -0,0 +1 @@
+<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -0,0 +1 @@
+<meta   charset = utf-8 />

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloc(size + 1);
+  char *dest = malloct(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,42 +549,6 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
-HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
-  /* text/html; charset=utf-8 */
-  const char *const charset = "charset";
-  char *pos = strstr(mime, charset);
-
-  if (pos != NULL) {
-    /* Skip spaces */
-    int eq = 0;
-
-    for(pos += strlen(charset);
-        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
-      if (*pos == '=') {
-        eq = 1;
-      }
-    }
-    if (eq == 1) {
-      int len;
-
-      for(len = 0;
-          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
-          pos++) ;
-      if (len != 0) {
-        char *const s = malloc(len + 1);
-        int i;
-
-        for(i = 0; i < len; i++) {
-          s[i] = pos[i];
-        }
-        s[len] = '\0';
-        return s;
-      }
-    }
-  }
-  return NULL;
-}
-
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -644,58 +608,106 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
+/* Extract X from a "text/html; charset=X" content= attribute value. */
+static char *charset_from_content(const char *s, size_t len) {
+  size_t i;
+
+  for (i = 0; i + 7 < len; i++) {
+    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
+        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
+      size_t j, val;
+
+      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
+        ;
+      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
+                    !is_space(s[j]);
+           j++)
+        ;
+      if (j != val) {
+        return hts_stringMemCopy(&s[val], j - val);
+      }
+    }
+  }
+  return NULL;
+}
+
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  int i;
+  size_t i;
 
-  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
-  for(i = 0; i < size; i++) {
-    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
-        && is_space(html[i + 5])) {
-      /* Skip spaces */
-      for(i += 5; is_space(html[i]); i++) ;
-      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
-          && is_space_or_equal(html[i + 10])) {
-        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
-        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
-          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
-          if (strncasecmp(&html[i], "CONTENT", 7) == 0
-              && is_space_or_equal(html[i + 7])) {
-            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
-            /* Skip content-type */
-            for(;
-                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
-                i++) ;
-            /* Expect charset attribute here */
-            if (html[i] == ';') {
-              for(i++; is_space(html[i]); i++) ;
-              /* Look for charset */
-              if (strncasecmp(&html[i], "charset", 7) == 0
-                  && is_space_or_equal(html[i + 7])) {
-                int len;
+  /* HTML5 <meta charset=X> or legacy
+     <meta http-equiv="Content-Type" content="text/html; charset=X">,
+     attributes in any order; first resolvable tag wins. */
+  for (i = 0; i + 6 < size; i++) {
+    size_t j;
+    const char *equiv = NULL, *content = NULL;
+    size_t equiv_len = 0, content_len = 0;
 
-                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
-                    i++) ;
-                /* Charset */
-                for(len = 0;
-                    i + len < size && html[i + len] != '"'
-                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
-                /* No error ? */
-                if (len != 0 && i < size) {
-                  char *const s = malloc(len + 1);
-                  int j;
+    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
+        !is_space(html[i + 5])) {
+      continue;
+    }
+    /* Attribute scan, strictly bounded by size. */
+    for (j = i + 6; j < size && html[j] != '>';) {
+      size_t name, name_len, val = 0, val_len = 0;
 
-                  for(j = 0; j < len; j++) {
-                    s[j] = html[i + j];
-                  }
-                  s[len] = '\0';
-                  return s;
-                }
-              }
-            }
+      if (is_space(html[j]) || html[j] == '/') {
+        j++;
+        continue;
+      }
+      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
+                     html[j] != '/' && !is_space(html[j]);
+           j++)
+        ;
+      name_len = j - name;
+      for (; j < size && is_space(html[j]); j++)
+        ;
+      if (j < size && html[j] == '=') {
+        for (j++; j < size && is_space(html[j]); j++)
+          ;
+        if (j < size && (html[j] == '"' || html[j] == '\'')) {
+          const char quote = html[j++];
+
+          for (val = j; j < size && html[j] != quote; j++)
+            ;
+          if (j >= size) /* unterminated quote: drop the tag */
+            break;
+          val_len = j++ - val;
+        } else {
+          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
+             belongs to the value except as the tail of a self-close */
+          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
+            ;
+          val_len = j - val;
+          if (val_len != 0 && j < size && html[j] == '>' &&
+              html[val + val_len - 1] == '/') {
+            val_len--;
           }
         }
       }
+      /* first occurrence of each attribute wins, as in browsers */
+      if (val_len != 0) {
+        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
+          return hts_stringMemCopy(&html[val], val_len);
+        } else if (equiv == NULL && name_len == 10 &&
+                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
+          equiv = &html[val];
+          equiv_len = val_len;
+        } else if (content == NULL && name_len == 7 &&
+                   strncasecmp(&html[name], "content", 7) == 0) {
+          content = &html[val];
+          content_len = val_len;
+        }
+      }
     }
+    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
+        content != NULL) {
+      char *const s = charset_from_content(content, content_len);
+
+      if (s != NULL) {
+        return s;
+      }
+    }
+    i = j;
   }
   return NULL;
 }
@@ -1140,17 +1152,45 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
+  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    /* Reader: can read bytes up to j */
-#define RD ( i < size ? data[i++] : -1 )
+    const unsigned char c = data[i];
+    size_t len, k;
+    hts_UCS4 uc, min;
 
-    /* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
-#define WR(C) if ((C) == -1) { return 0; }
+    if (c < 0x80) {
+      i++;
+      continue;
+    } else if (c >= 0xc2 && c <= 0xdf) {
+      len = 2;
+      min = 0x80;
+      uc = c & 0x1f;
+    } else if (c >= 0xe0 && c <= 0xef) {
+      len = 3;
+      min = 0x800;
+      uc = c & 0x0f;
+    } else if (c >= 0xf0 && c <= 0xf4) {
+      len = 4;
+      min = 0x10000;
+      uc = c & 0x07;
+    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
+      return 0;
+    }
+    if (size - i < len) {
+      return 0;
+    }
+    for (k = 1; k < len; k++) {
+      const unsigned char cc = data[i + k];
 
-    /* Read Unicode character. */
-    READ_UNICODE(RD, WR);
-#undef RD
-#undef WR
+      if ((cc & 0xc0) != 0x80) {
+        return 0;
+      }
+      uc = (uc << 6) | (cc & 0x3f);
+    }
+    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
+      return 0;
+    }
+    i += len;
   }
 
   return 1;

src/htscharset.h

@@ -76,7 +76,8 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the charset from the HTML buffer "html"
+ * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
+ * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -86,7 +87,8 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string an UTF-8 string ?
+ * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
+ * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsparse.c

@@ -2007,7 +2007,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                 /* Unescape/escape %20 and other   */
                 {
-                  // Note: always true (iso-8859-1 as default)
+                  // NULL when UTF-8 conversion is off (-%T0)
                   const char *const charset = str->page_charset_;
                   const int hasCharset = charset != NULL 
                     && *charset != '\0';
@@ -2030,11 +2030,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                   // Force to encode non-printable chars (should never happend)
                   escape_remove_control(lien);
-                  
-                  // charset conversion for the URI filename, 
-                  // and not already UTF-8
-                  // (note: not for the query string!)
-                  if (hasCharset && !hts_isCharsetUTF8(charset)) {
+
+                  // charset conversion for the URI filename (not the query
+                  // string), unless the bytes already are valid UTF-8:
+                  // converting those would double-encode them (#180)
+                  if (hasCharset && !hts_isCharsetUTF8(charset) &&
+                      !hts_isStringUTF8(lien, strlen(lien))) {
                     char *const s = hts_convertStringToUTF8(lien, strlen(lien), charset);
                     if (s != NULL) {
                       hts_log_print(opt, LOG_DEBUG,

src/htsselftest.c

@@ -788,6 +788,30 @@ static int st_charset(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static int st_metacharset(httrackp *opt, int argc, char **argv) {
+  char *s;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "metacharset: needs an html string\n");
+    return 1;
+  }
+  s = hts_getCharsetFromMeta(argv[0], strlen(argv[0]));
+  printf("%s\n", s != NULL ? s : "(none)");
+  freet(s);
+  return 0;
+}
+
+static int st_isutf8(httrackp *opt, int argc, char **argv) {
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "isutf8: needs a string\n");
+    return 1;
+  }
+  printf("%d\n", hts_isStringUTF8(argv[0], strlen(argv[0])) ? 1 : 0);
+  return 0;
+}
+
 static int st_idna_encode(httrackp *opt, int argc, char **argv) {
   char *s;
 
@@ -2394,6 +2418,9 @@ static const struct selftest_entry {
     {"mime", "<filename>", "MIME type for a filename", st_mime},
     {"charset", "<charset> <string>",
      "convert a string to UTF-8 from a charset", st_charset},
+    {"metacharset", "<html>", "extract the <meta> charset from an HTML page",
+     st_metacharset},
+    {"isutf8", "<string>", "is the string valid UTF-8 (1/0)", st_isutf8},
     {"idna-encode", "<host>", "encode a hostname to IDNA/punycode",
      st_idna_encode},
     {"idna-decode", "<host>", "decode an IDNA/punycode hostname",

tests/01_engine-charset.test

@@ -36,3 +36,64 @@ test "$(httrack -O /dev/null -#test=charset 'no-such-charset-xyz' 'café' 2>/dev
 # malformed UTF-8 (lone continuation byte, truncated lead byte) must not crash
 runs 'utf-8' $'\x80'
 runs 'utf-8' $'\xc3'
+
+# hts_getCharsetFromMeta: <meta> charset extraction, HTML5 and legacy forms.
+# -#test=metacharset <html> prints the charset, or "(none)".
+meta() {
+    test "$(httrack -O /dev/null -#test=metacharset "$1")" == "$2" || exit 1
+}
+
+# HTML5 form, quoting/spacing flavors
+meta '<meta charset="utf-8">' 'utf-8'
+meta "<meta charset='utf-8'>" 'utf-8'
+meta '<meta charset=utf-8>' 'utf-8'
+meta '<meta charset=utf-8 />' 'utf-8'
+meta '<meta charset=utf-8/>' 'utf-8'
+# charset labels are case-insensitive downstream; only require case-insensitive match
+test "$(httrack -O /dev/null -#test=metacharset '<META CHARSET=UTF-8>' | tr '[:upper:]' '[:lower:]')" == 'utf-8' || exit 1
+meta '<html><head><meta   charset = "utf-8"  ></head>' 'utf-8'
+
+# legacy http-equiv form, attributes in any order
+meta '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' 'iso-8859-1'
+meta '<meta content="text/html; charset=gb2312" http-equiv="content-type">' 'gb2312'
+meta '<meta http-equiv=Content-Type content="text/html;charset=euc-jp">' 'euc-jp'
+meta '<meta http-equiv="Content-Type" content="text/html; charset = koi8-r ">' 'koi8-r'
+meta '<meta http-equiv="Content-Type" content="text/html; charset=utf-8;">' 'utf-8'
+meta '<meta http-equiv=content-type content=text/html;charset=utf-8 />' 'utf-8'
+meta '<meta http-equiv=content-type content=text/html;charset=utf-8>' 'utf-8'
+
+# first resolvable meta wins; unrelated metas are skipped
+meta '<meta name="viewport" content="width=1"><meta charset="koi8-r">' 'koi8-r'
+meta '<meta charset="utf-8"><meta http-equiv="Content-Type" content="text/html; charset=gb2312">' 'utf-8'
+
+# duplicate attribute: first wins, as in browsers
+meta '<meta http-equiv="Content-Type" content="text/html; charset=big5" http-equiv="refresh">' 'big5'
+
+# negatives: wrong tag/attribute, empty value
+meta '<p>x</p>' '(none)'
+meta '<meta name="desc" content="charset=bogus">' '(none)'
+meta '<meta http-equiv="refresh" content="0; url=x.html">' '(none)'
+meta '<body charset="utf-8">' '(none)'
+meta '<meta charset="">' '(none)'
+
+# hostile truncations: no crash, no match
+meta '<meta charset="utf-8' '(none)'
+meta '<meta charset=' '(none)'
+meta '<meta ' '(none)'
+meta '<meta' '(none)'
+
+# hts_isStringUTF8: valid-UTF-8 probe (guards the #180 link conversion).
+utf8() {
+    test "$(httrack -O /dev/null -#test=isutf8 "$1")" == "$2" || exit 1
+}
+utf8 'plain' '1'
+utf8 'café' '1'
+utf8 '统计' '1'
+utf8 "$(printf 'caf\xe9')" '0'              # latin-1 accent
+utf8 "$(printf '\xc3')" '0'                 # truncated lead byte
+utf8 "$(printf '\xa9')" '0'                 # lone continuation byte
+utf8 "$(printf '\xf0\x9f\x98\x80')" '1'     # 4-byte emoji
+utf8 "$(printf '\xc0\xaf')" '0'             # overlong (latin-1 "À¯" must stay convertible)
+utf8 "$(printf '\xed\xa0\x80')" '0'         # UTF-16 surrogate
+utf8 "$(printf '\xf4\x90\x80\x80')" '0'     # above U+10FFFF
+utf8 "$(printf '\xf8\x88\x80\x80\x80')" '0' # legacy 5-byte form

tests/11_crawl-international.test

@@ -22,9 +22,9 @@ bash crawl-test.sh \
     httrack http://ut.httrack.com/unicode-links/utf8.html
 
 bash crawl-test.sh \
-    --errors 4 --files 7 \
-    --found ut.httrack.com/unicode-links/café3860.html \
-    --found ut.httrack.com/unicode-links/café9fa8.html \
+    --errors 2 --files 9 \
+    --found ut.httrack.com/unicode-links/café3860.html \
+    --found ut.httrack.com/unicode-links/café9fa8.html \
     --found ut.httrack.com/unicode-links/café30f4.html \
     --found ut.httrack.com/unicode-links/café5e1f.html \
     --found ut.httrack.com/unicode-links/café7b30.html \

tests/41_local-utf8-link.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# #180: raw non-ASCII hrefs must reach the wire UTF-8 percent-encoded whatever
+# the declared page charset; each PDF exists only at its exact UTF-8 path.
+
+set -euo pipefail
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
+    --found 'charset/header/统计大数据服务平台.pdf' \
+    --found 'charset/meta5/统计大数据服务平台.pdf' \
+    --found 'charset/metaeq/统计大数据服务平台.pdf' \
+    --found 'charset/none/统计大数据服务平台.pdf' \
+    --found 'charset/latin1hdr/统计大数据服务平台.pdf' \
+    --found 'charset/latin1real/café.pdf' \
+    --found 'charset/metalatin1/déjà.pdf' \
+    --found 'charset/latin1ovl/À¡x.pdf' \
+    --found 'charset/priority/nuée.pdf' \
+    --found 'charset/preenc/统计大数据服务平台.pdf' \
+    --found 'charset/bom/统计大数据服务平台.pdf' \
+    httrack 'BASEURL/charset/index.html'

tests/Makefile.am

@@ -109,6 +109,7 @@ TESTS = \
 	37_local-cache-outage.test \
 	38_local-update-304.test \
 	39_local-delayed-cancel.test \
-	40_local-why.test
+	40_local-why.test \
+	41_local-utf8-link.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -660,6 +660,97 @@ class Handler(SimpleHTTPRequestHandler):
     def route_intl_page(self):
         self.send_raw(b"<html><body>accented page</body></html>\n", "text/html")
 
+    # Raw non-ASCII href matrix (#180): each variant declares the page charset
+    # differently; the PDF exists only at its exact UTF-8 path, so a
+    # mis-decoded link 404s.
+    CHARSET_CJK = "统计大数据服务平台.pdf"
+    # variant -> (index Content-Type, <head> bytes, href bytes, pdf name)
+    CHARSET_VARIANTS = {
+        "header": ("text/html; charset=utf-8", b"", CHARSET_CJK.encode(), CHARSET_CJK),
+        "meta5": (
+            "text/html",
+            b'<meta charset="utf-8">',
+            CHARSET_CJK.encode(),
+            CHARSET_CJK,
+        ),
+        "metaeq": (
+            "text/html",
+            b'<meta http-equiv="Content-Type" content="text/html; charset=utf-8">',
+            CHARSET_CJK.encode(),
+            CHARSET_CJK,
+        ),
+        "none": ("text/html", b"", CHARSET_CJK.encode(), CHARSET_CJK),
+        "latin1hdr": (
+            "text/html; charset=iso-8859-1",
+            b"",
+            CHARSET_CJK.encode(),
+            CHARSET_CJK,
+        ),
+        # genuine latin-1 href: the charset conversion must still apply
+        "latin1real": (
+            "text/html; charset=iso-8859-1",
+            b"",
+            "café.pdf".encode("latin-1"),
+            "café.pdf",
+        ),
+        # latin-1 declared by META only: the meta parser is load-bearing
+        "metalatin1": (
+            "text/html",
+            b'<meta charset="iso-8859-1">',
+            "déjà.pdf".encode("latin-1"),
+            "déjà.pdf",
+        ),
+        # latin-1 bytes that form an overlong UTF-8 shape: strict validation
+        # must still convert them
+        "latin1ovl": (
+            "text/html; charset=iso-8859-1",
+            b"",
+            "À¡x.pdf".encode("latin-1"),
+            "À¡x.pdf",
+        ),
+        # header wins over meta: latin-1 href only resolves if iso-8859-1 is kept
+        "priority": (
+            "text/html; charset=iso-8859-1",
+            b'<meta charset="utf-8">',
+            "nuée.pdf".encode("latin-1"),
+            "nuée.pdf",
+        ),
+        "preenc": ("text/html", b"", quote(CHARSET_CJK).encode(), CHARSET_CJK),
+        "bom": ("text/html", b"", CHARSET_CJK.encode(), CHARSET_CJK),
+    }
+
+    def route_charset(self):
+        path = unquote(urlsplit(self.path).path)
+        parts = path.split("/")
+        if path == "/charset/index.html":
+            self.send_html(
+                "".join(
+                    '\t<a href="%s/index.html">%s</a>\n' % (v, v)
+                    for v in self.CHARSET_VARIANTS
+                )
+            )
+            return
+        if len(parts) == 4 and parts[2] in self.CHARSET_VARIANTS:
+            ctype, head, href, pdf = self.CHARSET_VARIANTS[parts[2]]
+            if parts[3] == "index.html":
+                body = (
+                    b"<html><head>"
+                    + head
+                    + b'</head><body><a href="'
+                    + href
+                    + b'">doc</a></body></html>'
+                )
+                if parts[2] == "bom":
+                    body = b"\xef\xbb\xbf" + body
+                self.send_raw(body, ctype)
+                return
+            if parts[3] == pdf:
+                self.send_raw(self.FAKE_PDF, "application/pdf")
+                return
+        self.send_response(404)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
     # resume / 416 loop (#206): the first GET stalls after a prefix so the crawl
     # can be interrupted (partial + temp-ref); every later request is 416.
     RESUME_PREFIX = b"PARTIAL-" + b"x" * 4096  # flushed before the stall
@@ -1194,6 +1285,9 @@ class Handler(SimpleHTTPRequestHandler):
         if path.startswith("/big/"):
             self.route_big()
             return True
+        if path.startswith("/charset/"):
+            self.route_charset()
+            return True
         # Match percent-encoded paths (accented #157 route) by their decoded form.
         handler = self.ROUTES.get(path) or self.ROUTES.get(unquote(path))
         if handler is not None: