Drop the per-request pre-resolve that slowed crawls on non-default ports (#181)

back_solve() resolved the raw "host:port" authority once per backing
request through check_hostname_dns() — an uncached getaddrinfo whose
result was only logged (host_wait() is always synchronous now). The
connect path already strips the port and resolves through the DNS cache,
so this lookup was pure overhead. On resolvers that don't fail fast on
the malformed "host:port" name (search-domain expansion, DNS timeouts)
it added a per-request stall: the reported slowness. Mirrors on port 80,
or with the port omitted, avoided the bad name and stayed fast.

Remove back_solve() and its debug-only lookup; keep the synchronous
host_wait() gate. Add a DNS self-test pinning that the cache/connect
path resolves "host:port" as the bare host.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
-# installed headers (no struct layout or exported signature changed vs
-# 3.49.10), so it stays soname .so.3; bump revision.
+# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
+# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
+# so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:3:0"
+VERSION_INFO="3:4:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

debian/changelog

@@ -1,3 +1,13 @@
+httrack (3.49.12-1) unstable; urgency=medium
+
+  * New upstream release: security and crawl-correctness fixes (remote stack
+    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
+    world-readable cookies.txt, filter-pattern denial of service) plus a new
+    --why filter diagnostic; full list in history.txt.
+  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
+
+ -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
+
 httrack (3.49.11-1) unstable; urgency=medium
 
   * New upstream release: crawl correctness and security fixes (network-facing

fuzz/corpus/meta/meta-content-first.html

@@ -0,0 +1 @@
+<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-truncated.html

@@ -0,0 +1 @@
+<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -0,0 +1 @@
+<meta   charset = utf-8 />

history.txt

@@ -4,6 +4,23 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-12
++ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
++ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
++ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
++ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
++ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
++ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
++ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
++ Fixed: cookies.txt was created world-readable (#511)
++ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
++ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
++ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
++ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
++ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
++ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
++ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
+
 3.49-11
 + New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
 + New: advertise deflate in Accept-Encoding and decode deflate responses (#450)

src/htsback.c

@@ -1646,138 +1646,104 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       }
     }
     // tester cache
-    if ((strcmp(adr, "file://"))        /* pas fichier */
-        &&((!test) || (cache->type == 1))       /* cache prioritaire, laisser passer en test! */
-        &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
-#if HTS_FAST_CACHE
+    if ((strcmp(adr, "file://")) /* pas fichier */
+        && ((!test) ||
+            (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
+        &&
+        ((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
+                                 0))) { // si en test on ne doit pas utiliser le
+                                        // cache sinon telescopage avec le 302..
       intptr_t hash_pos;
       int hash_pos_return = 0;
-#else
-      char *a = NULL;
-#endif
-#if HTS_FAST_CACHE
+
       if (cache->hashtable) {
-#else
-      if (cache->use) {
-#endif
         char BIGSTK buff[HTS_URLMAXSIZE * 4];
 
-#if HTS_FAST_CACHE
         strcpybuff(buff, adr);
         strcatbuff(buff, fil);
         hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
-#else
-        buff[0] = '\0';
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, adr);
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, fil);
-        strcatbuff(buff, "\n");
-        a = strstr(cache->use, buff);
-#endif
 
-        // Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
-#if HTS_FAST_CACHE
         // negative values when data is not in cache
         if (hash_pos_return < 0) {
-#else
-        if (a) {
-#endif
-          if (!test) {          // non mode test
-#if HTS_FAST_CACHE==0
-            int pos = -1;
+          if (!test) { // not test mode
+            /* note: no check with IS_DELAYED_EXT() enabled - postcheck by
+             * client please! */
+            if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
+                fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
+                    0) { // final file missing or empty
+              int found = 0;
 
-            a += strlen(buff);
-            sscanf(a, "%d", &pos);      // lire position
-#endif
+              /* It is possible that the file has been moved due to changes in
+               * build structure */
+              {
+                char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+                htsblk r;
 
-#if HTS_FAST_CACHE==0
-            if (pos < 0) {      // pas de mise en cache data, vérifier existence
-#endif
-              /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
-              if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) {  // fichier final n'existe pas ou est vide!
-                int found = 0;
-
-                /* It is possible that the file has been moved due to changes in build structure */
-                {
-                  char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
-                  htsblk r;
-
-                  previous_save[0] = '\0';
-                  r =
-                    cache_readex(opt, cache, adr, fil, /*head */ NULL,
+                previous_save[0] = '\0';
+                r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
                                  /*bound to back[p] (temporary) */
                                  back[p].location_buffer, previous_save, /*ro */
                                  1);
-                  /* Is supposed to be on disk only */
-                  if (r.is_write && previous_save[0] != '\0') {
-                    /* Exists, but with another (old) filename: rename (almost) silently */
-                    if (strcmp(previous_save, save) != 0
-                        && fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
-                      rename(fconv(catbuff, sizeof(catbuff), previous_save),
-                             fconv(catbuff2, sizeof(catbuff2), save));
-                      if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
-                        found = 1;
-                        hts_log_print(opt, LOG_DEBUG,
-                                      "File '%s' has been renamed since last mirror to '%s' ; applying changes",
-                                      previous_save, save);
-                      } else {
-                        hts_log_print(opt, LOG_ERROR,
-                                      "Could not rename '%s' to '%s' ; will have to retransfer it",
-                                      previous_save, save);
-                      }
+                /* Is supposed to be on disk only */
+                if (r.is_write && previous_save[0] != '\0') {
+                  /* Exists, but with another (old) filename: rename (almost)
+                   * silently */
+                  if (strcmp(previous_save, save) != 0 &&
+                      fexist_utf8(
+                          fconv(catbuff, sizeof(catbuff), previous_save))) {
+                    rename(fconv(catbuff, sizeof(catbuff), previous_save),
+                           fconv(catbuff2, sizeof(catbuff2), save));
+                    if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
+                      found = 1;
+                      hts_log_print(opt, LOG_DEBUG,
+                                    "File '%s' has been renamed since last "
+                                    "mirror to '%s' ; applying changes",
+                                    previous_save, save);
+                    } else {
+                      hts_log_print(opt, LOG_ERROR,
+                                    "Could not rename '%s' to '%s' ; will have "
+                                    "to retransfer it",
+                                    previous_save, save);
                     }
                   }
-                  back[p].location_buffer[0] = '\0';
                 }
+                back[p].location_buffer[0] = '\0';
+              }
 
-                /* Not found ? */
-                if (!found) {
-#if HTS_FAST_CACHE
-                  hash_pos_return = 0;
-#else
-                  a = NULL;
-#endif
-                  // dévalider car non présent sur disque dans structure originale!!!
-                  // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
-                  // en être sûr
-                  if (opt->norecatch) { // tester norecatch
-                    if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {   // fichier existe pas mais déclaré: on l'a effacé
-                      FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
+              /* Not found ? */
+              if (!found) {
+                // invalidate: gone from disk, force a refetch
+                hash_pos_return = 0;
+                if (opt->norecatch) {
+                  if (!fexist_utf8(fconv(
+                          catbuff, sizeof(catbuff),
+                          save))) { // declared but missing: user erased it
+                    FILE *fp =
+                        FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
 
-                      if (fp)
-                        fclose(fp);
-                      hts_log_print(opt, LOG_WARNING,
-                                    "Previous file '%s' not found (erased by user ?), ignoring: %s%s",
-                                    save, back[p].url_adr, back[p].url_fil);
-                    }
-                  } else {
+                    if (fp)
+                      fclose(fp);
                     hts_log_print(opt, LOG_WARNING,
-                                  "Previous file '%s' not found (erased by user ?), recatching: %s%s",
+                                  "Previous file '%s' not found (erased by "
+                                  "user ?), ignoring: %s%s",
                                   save, back[p].url_adr, back[p].url_fil);
                   }
+                } else {
+                  hts_log_print(opt, LOG_WARNING,
+                                "Previous file '%s' not found (erased by user "
+                                "?), recatching: %s%s",
+                                save, back[p].url_adr, back[p].url_fil);
                 }
-              }                 // fsize() <= 0
-#if HTS_FAST_CACHE==0
-            }
-#endif
+              }
+            } // fsize() <= 0
           }
         }
         //
       } else {
-#if HTS_FAST_CACHE
         hash_pos_return = 0;
-#else
-        a = NULL;
-#endif
       }
 
-      // Existe pas en cache, ou bien pas de cache présent
-#if HTS_FAST_CACHE
-      if (hash_pos_return) {    // OK existe en cache (et données aussi)!
-#else
-      if (a != NULL) {          // OK existe en cache (et données aussi)!
-#endif
+      if (hash_pos_return) { // in cache, with data
         const int cache_is_prioritary = cache->type == 1
           || opt->state.stop != 0;
         if (cache_is_prioritary) {      // cache prioritaire (pas de test if-modified..)
@@ -1898,7 +1864,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #endif
         }
       }
-      /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
+      /* Not in cache ; maybe in temporary cache ? Warning: non-movable
+         "url_sav" */
       else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
         const off_t file_size = fsize_utf8(itemback->url_sav);
 
@@ -2083,13 +2050,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-#if HDEBUG
-        printf("back_solve..\n");
-#endif
-        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
-        soc = INVALID_SOCKET;   // pas encore ouverte
-        back_solve(opt, &back[p]);      // préparer
-        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
+        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
+        soc = INVALID_SOCKET;             // not opened yet
+        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2232,37 +2195,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// attendre que le host (ou celui du proxy) ait été résolu
-// si c'est un fichier, la résolution est immédiate
-// idem pour ftp://
-void back_solve(httrackp * opt, lien_back * back) {
-  assertf(opt != NULL);
-  assertf(back != NULL);
-  if ((!strfield(back->url_adr, "file://"))
-      && !strfield(back->url_adr, "ftp://")
-    ) {
-    const char *a;
-
-    if (!(back->r.req.proxy.active))
-      a = back->url_adr;
-    else
-      a = back->r.req.proxy.name;
-    assertf(a != NULL);
-    a = jump_protocol_const(a);
-    if (check_hostname_dns(a)) {
-      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
-    } else {
-      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
-    }
-  }
-}
-
-// détermine si le host a pu être résolu
-int host_wait(httrackp * opt, lien_back * back) {
-  // Always synchronous. No more background DNS resolution
-  // (does not really improve performances)
-  return 1;
-}
+// Resolution is synchronous inside the connect path; no pre-resolve step, so
+// the host is always immediately ready.
+int host_wait(httrackp *opt, lien_back *back) { return 1; }
 #endif
 
 // élimine les fichiers non html en backing (anticipation)

src/htsback.h

@@ -138,7 +138,6 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
-void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);

src/htscache.c

@@ -491,11 +491,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
   hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
   /* avoid errors on data entries */
   if (adr[0] == '/' && adr[1] == '/' && adr[2] == '[') {
-#if HTS_FAST_CACHE
     hash_pos_return = 0;
-#else
-    a = NULL;
-#endif
   }
 
   if (hash_pos_return != 0) {

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloc(size + 1);
+  char *dest = malloct(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,42 +549,6 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
-HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
-  /* text/html; charset=utf-8 */
-  const char *const charset = "charset";
-  char *pos = strstr(mime, charset);
-
-  if (pos != NULL) {
-    /* Skip spaces */
-    int eq = 0;
-
-    for(pos += strlen(charset);
-        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
-      if (*pos == '=') {
-        eq = 1;
-      }
-    }
-    if (eq == 1) {
-      int len;
-
-      for(len = 0;
-          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
-          pos++) ;
-      if (len != 0) {
-        char *const s = malloc(len + 1);
-        int i;
-
-        for(i = 0; i < len; i++) {
-          s[i] = pos[i];
-        }
-        s[len] = '\0';
-        return s;
-      }
-    }
-  }
-  return NULL;
-}
-
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -644,58 +608,106 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
+/* Extract X from a "text/html; charset=X" content= attribute value. */
+static char *charset_from_content(const char *s, size_t len) {
+  size_t i;
+
+  for (i = 0; i + 7 < len; i++) {
+    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
+        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
+      size_t j, val;
+
+      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
+        ;
+      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
+                    !is_space(s[j]);
+           j++)
+        ;
+      if (j != val) {
+        return hts_stringMemCopy(&s[val], j - val);
+      }
+    }
+  }
+  return NULL;
+}
+
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  int i;
+  size_t i;
 
-  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
-  for(i = 0; i < size; i++) {
-    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
-        && is_space(html[i + 5])) {
-      /* Skip spaces */
-      for(i += 5; is_space(html[i]); i++) ;
-      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
-          && is_space_or_equal(html[i + 10])) {
-        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
-        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
-          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
-          if (strncasecmp(&html[i], "CONTENT", 7) == 0
-              && is_space_or_equal(html[i + 7])) {
-            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
-            /* Skip content-type */
-            for(;
-                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
-                i++) ;
-            /* Expect charset attribute here */
-            if (html[i] == ';') {
-              for(i++; is_space(html[i]); i++) ;
-              /* Look for charset */
-              if (strncasecmp(&html[i], "charset", 7) == 0
-                  && is_space_or_equal(html[i + 7])) {
-                int len;
+  /* HTML5 <meta charset=X> or legacy
+     <meta http-equiv="Content-Type" content="text/html; charset=X">,
+     attributes in any order; first resolvable tag wins. */
+  for (i = 0; i + 6 < size; i++) {
+    size_t j;
+    const char *equiv = NULL, *content = NULL;
+    size_t equiv_len = 0, content_len = 0;
 
-                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
-                    i++) ;
-                /* Charset */
-                for(len = 0;
-                    i + len < size && html[i + len] != '"'
-                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
-                /* No error ? */
-                if (len != 0 && i < size) {
-                  char *const s = malloc(len + 1);
-                  int j;
+    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
+        !is_space(html[i + 5])) {
+      continue;
+    }
+    /* Attribute scan, strictly bounded by size. */
+    for (j = i + 6; j < size && html[j] != '>';) {
+      size_t name, name_len, val = 0, val_len = 0;
 
-                  for(j = 0; j < len; j++) {
-                    s[j] = html[i + j];
-                  }
-                  s[len] = '\0';
-                  return s;
-                }
-              }
-            }
+      if (is_space(html[j]) || html[j] == '/') {
+        j++;
+        continue;
+      }
+      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
+                     html[j] != '/' && !is_space(html[j]);
+           j++)
+        ;
+      name_len = j - name;
+      for (; j < size && is_space(html[j]); j++)
+        ;
+      if (j < size && html[j] == '=') {
+        for (j++; j < size && is_space(html[j]); j++)
+          ;
+        if (j < size && (html[j] == '"' || html[j] == '\'')) {
+          const char quote = html[j++];
+
+          for (val = j; j < size && html[j] != quote; j++)
+            ;
+          if (j >= size) /* unterminated quote: drop the tag */
+            break;
+          val_len = j++ - val;
+        } else {
+          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
+             belongs to the value except as the tail of a self-close */
+          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
+            ;
+          val_len = j - val;
+          if (val_len != 0 && j < size && html[j] == '>' &&
+              html[val + val_len - 1] == '/') {
+            val_len--;
           }
         }
       }
+      /* first occurrence of each attribute wins, as in browsers */
+      if (val_len != 0) {
+        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
+          return hts_stringMemCopy(&html[val], val_len);
+        } else if (equiv == NULL && name_len == 10 &&
+                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
+          equiv = &html[val];
+          equiv_len = val_len;
+        } else if (content == NULL && name_len == 7 &&
+                   strncasecmp(&html[name], "content", 7) == 0) {
+          content = &html[val];
+          content_len = val_len;
+        }
+      }
     }
+    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
+        content != NULL) {
+      char *const s = charset_from_content(content, content_len);
+
+      if (s != NULL) {
+        return s;
+      }
+    }
+    i = j;
   }
   return NULL;
 }
@@ -1140,17 +1152,45 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
+  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    /* Reader: can read bytes up to j */
-#define RD ( i < size ? data[i++] : -1 )
+    const unsigned char c = data[i];
+    size_t len, k;
+    hts_UCS4 uc, min;
 
-    /* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
-#define WR(C) if ((C) == -1) { return 0; }
+    if (c < 0x80) {
+      i++;
+      continue;
+    } else if (c >= 0xc2 && c <= 0xdf) {
+      len = 2;
+      min = 0x80;
+      uc = c & 0x1f;
+    } else if (c >= 0xe0 && c <= 0xef) {
+      len = 3;
+      min = 0x800;
+      uc = c & 0x0f;
+    } else if (c >= 0xf0 && c <= 0xf4) {
+      len = 4;
+      min = 0x10000;
+      uc = c & 0x07;
+    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
+      return 0;
+    }
+    if (size - i < len) {
+      return 0;
+    }
+    for (k = 1; k < len; k++) {
+      const unsigned char cc = data[i + k];
 
-    /* Read Unicode character. */
-    READ_UNICODE(RD, WR);
-#undef RD
-#undef WR
+      if ((cc & 0xc0) != 0x80) {
+        return 0;
+      }
+      uc = (uc << 6) | (cc & 0x3f);
+    }
+    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
+      return 0;
+    }
+    i += len;
   }
 
   return 1;

src/htscharset.h

@@ -76,7 +76,8 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the charset from the HTML buffer "html"
+ * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
+ * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -86,7 +87,8 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string an UTF-8 string ?
+ * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
+ * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsconfig.h

@@ -76,9 +76,6 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// fast cache (build hash table)
-#define HTS_FAST_CACHE 1
-
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1

src/htscore.c

@@ -134,10 +134,6 @@ RUN_CALLBACK0(opt, end); \
     back_delete_all(opt, &cache, sback);                                       \
     back_free(&sback);                                                         \
     checkrobots_free(&robots);                                                 \
-    if (cache.use) {                                                           \
-      freet(cache.use);                                                        \
-      cache.use = NULL;                                                        \
-    }                                                                          \
     if (cache.zipOutput) {                                                     \
       zipClose(cache.zipOutput,                                                \
                "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \

src/htscore.h

@@ -194,7 +194,6 @@ struct cache_back {
   int type;
   int ro;                   /**< read-only: no new cache is written */
 
-  char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];

src/htsdns_selftest.c

@@ -297,6 +297,26 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
+  /* "host:port" resolves as the bare host: the cache/connect path strips
+     ":port" before both the backend and the cache key (#181). */
+  mock_reset_calls();
+  {
+    SOCaddr a;
+    char ip[64];
+    const char *err = NULL;
+
+    /* cache miss: the backend is hit with the stripped host, not "host:port" */
+    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* the bare host shares that one cache entry (stripped key) */
+    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* a port variant of an already-cached host also hits, right address */
+    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
+    SOCaddr_inetntoa(ip, sizeof(ip), a);
+    CHECK(strcmp(ip, "1.2.3.4") == 0);
+  }
+
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-11"
-#define HTTRACK_VERSIONID "3.49.11"
+#define HTTRACK_VERSION "3.49-12"
+#define HTTRACK_VERSIONID "3.49.12"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 

src/htsparse.c

@@ -2007,7 +2007,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                 /* Unescape/escape %20 and other   */
                 {
-                  // Note: always true (iso-8859-1 as default)
+                  // NULL when UTF-8 conversion is off (-%T0)
                   const char *const charset = str->page_charset_;
                   const int hasCharset = charset != NULL 
                     && *charset != '\0';
@@ -2030,11 +2030,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                   // Force to encode non-printable chars (should never happend)
                   escape_remove_control(lien);
-                  
-                  // charset conversion for the URI filename, 
-                  // and not already UTF-8
-                  // (note: not for the query string!)
-                  if (hasCharset && !hts_isCharsetUTF8(charset)) {
+
+                  // charset conversion for the URI filename (not the query
+                  // string), unless the bytes already are valid UTF-8:
+                  // converting those would double-encode them (#180)
+                  if (hasCharset && !hts_isCharsetUTF8(charset) &&
+                      !hts_isStringUTF8(lien, strlen(lien))) {
                     char *const s = hts_convertStringToUTF8(lien, strlen(lien), charset);
                     if (s != NULL) {
                       hts_log_print(opt, LOG_DEBUG,

src/htsselftest.c

@@ -788,6 +788,30 @@ static int st_charset(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static int st_metacharset(httrackp *opt, int argc, char **argv) {
+  char *s;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "metacharset: needs an html string\n");
+    return 1;
+  }
+  s = hts_getCharsetFromMeta(argv[0], strlen(argv[0]));
+  printf("%s\n", s != NULL ? s : "(none)");
+  freet(s);
+  return 0;
+}
+
+static int st_isutf8(httrackp *opt, int argc, char **argv) {
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "isutf8: needs a string\n");
+    return 1;
+  }
+  printf("%d\n", hts_isStringUTF8(argv[0], strlen(argv[0])) ? 1 : 0);
+  return 0;
+}
+
 static int st_idna_encode(httrackp *opt, int argc, char **argv) {
   char *s;
 
@@ -2123,6 +2147,19 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
+/* Write src[0..len) to path as-is; 0 on success. */
+static int ae_write_raw(const char *path, const unsigned char *src,
+                        size_t len) {
+  FILE *const f = FOPEN(path, "wb");
+  int ok;
+
+  if (f == NULL)
+    return 1;
+  ok = fwrite(src, 1, len, f) == len;
+  fclose(f);
+  return ok ? 0 : 1;
+}
+
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -2190,6 +2227,46 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
+    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
+       verbatim, small and multi-chunk (> one 8 KiB fread). */
+    assertf(ae_write_raw(inpath, small, slen) == 0);
+    assertf(hts_zunpack(inpath, outpath) == (int) slen);
+    assertf(ae_check_decoded(outpath, small, slen) == 0);
+    {
+      const size_t ilen = 16 * 1024;
+      unsigned char *idbody = malloct(ilen);
+
+      assertf(idbody != NULL);
+      for (i = 0; i < ilen; i++)
+        idbody[i] = small[i % slen];
+      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
+      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
+      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
+      freet(idbody);
+    }
+    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
+       must all still fail, not fall back to a verbatim copy. */
+    {
+      static const struct {
+        int wb;
+        size_t cut;
+      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
+
+      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
+        unsigned char z[512];
+        size_t zlen;
+        FILE *f;
+
+        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
+        f = FOPEN(inpath, "rb");
+        assertf(f != NULL);
+        zlen = fread(z, 1, sizeof(z), f);
+        fclose(f);
+        assertf(zlen > tr[i].cut && zlen < sizeof(z));
+        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
+        assertf(hts_zunpack(inpath, outpath) < 0);
+      }
+    }
     freet(body);
   }
 #else
@@ -2394,6 +2471,9 @@ static const struct selftest_entry {
     {"mime", "<filename>", "MIME type for a filename", st_mime},
     {"charset", "<charset> <string>",
      "convert a string to UTF-8 from a charset", st_charset},
+    {"metacharset", "<html>", "extract the <meta> charset from an HTML page",
+     st_metacharset},
+    {"isutf8", "<string>", "is the string valid UTF-8 (1/0)", st_isutf8},
     {"idna-encode", "<host>", "encode a hostname to IDNA/punycode",
      st_idna_encode},
     {"idna-decode", "<host>", "decode an IDNA/punycode hostname",

src/htszlib.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 
 /*
   Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
+  A body provably in no deflate framing is copied verbatim (identity).
   Return value: size of the new file, or -1 if an error occurred
 */
 /* Note: utf-8 */
@@ -68,6 +69,10 @@ int hts_zunpack(char *filename, char *newfile) {
             ((inbuf[0] & 0x0f) == Z_DEFLATED &&
              (((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
       int attempt;
+      /* not_deflate: the raw attempt hit a data error, so the body is in no
+         deflate framing at all. env_error: local I/O or memory failure. */
+      hts_boolean not_deflate = HTS_FALSE;
+      hts_boolean env_error = HTS_FALSE;
 
       /* deflate is ambiguous; on failure retry with the other windowBits */
       for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
@@ -78,15 +83,20 @@ int hts_zunpack(char *filename, char *newfile) {
 
         if (attempt > 0) {
           /* rewind input; reopening fpout "wb" discards the partial output */
-          if (fseek(in, 0, SEEK_SET) != 0)
+          if (fseek(in, 0, SEEK_SET) != 0) {
+            env_error = HTS_TRUE;
             break;
+          }
           navail = fread(inbuf, 1, sizeof(inbuf), in);
         }
         fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-        if (fpout == NULL)
+        if (fpout == NULL) {
+          env_error = HTS_TRUE;
           break;
+        }
         memset(&strm, 0, sizeof(strm));
         if (inflateInit2(&strm, windowBits) != Z_OK) {
+          env_error = HTS_TRUE;
           fclose(fpout);
           break;
         }
@@ -108,12 +118,17 @@ int hts_zunpack(char *filename, char *newfile) {
               zerr = inflate(&strm, Z_NO_FLUSH);
               if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
                   zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
+                if (zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR)
+                  env_error = HTS_TRUE;
+                else if (windowBits < 0)
+                  not_deflate = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
               produced = sizeof(outbuf) - strm.avail_out;
               if (produced > 0 &&
                   fwrite(outbuf, 1, produced, fpout) != produced) {
+                env_error = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
@@ -129,6 +144,28 @@ int hts_zunpack(char *filename, char *newfile) {
         inflateEnd(&strm);
         fclose(fpout);
       }
+      /* keep a mislabeled identity body verbatim only when provably not
+         deflate; truncation or a local failure must keep failing (#47) */
+      if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
+        FILE *const fpout =
+            FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
+
+        if (fpout != NULL && fseek(in, 0, SEEK_SET) == 0) {
+          int size = 0;
+
+          while ((navail = fread(inbuf, 1, sizeof(inbuf), in)) > 0) {
+            if (fwrite(inbuf, 1, navail, fpout) != navail) {
+              size = -1;
+              break;
+            }
+            size += (int) navail;
+          }
+          if (size >= 0 && !ferror(in))
+            ret = size;
+        }
+        if (fpout != NULL)
+          fclose(fpout);
+      }
       fclose(in);
     }
   }

tests/01_engine-charset.test

@@ -36,3 +36,64 @@ test "$(httrack -O /dev/null -#test=charset 'no-such-charset-xyz' 'café' 2>/dev
 # malformed UTF-8 (lone continuation byte, truncated lead byte) must not crash
 runs 'utf-8' $'\x80'
 runs 'utf-8' $'\xc3'
+
+# hts_getCharsetFromMeta: <meta> charset extraction, HTML5 and legacy forms.
+# -#test=metacharset <html> prints the charset, or "(none)".
+meta() {
+    test "$(httrack -O /dev/null -#test=metacharset "$1")" == "$2" || exit 1
+}
+
+# HTML5 form, quoting/spacing flavors
+meta '<meta charset="utf-8">' 'utf-8'
+meta "<meta charset='utf-8'>" 'utf-8'
+meta '<meta charset=utf-8>' 'utf-8'
+meta '<meta charset=utf-8 />' 'utf-8'
+meta '<meta charset=utf-8/>' 'utf-8'
+# charset labels are case-insensitive downstream; only require case-insensitive match
+test "$(httrack -O /dev/null -#test=metacharset '<META CHARSET=UTF-8>' | tr '[:upper:]' '[:lower:]')" == 'utf-8' || exit 1
+meta '<html><head><meta   charset = "utf-8"  ></head>' 'utf-8'
+
+# legacy http-equiv form, attributes in any order
+meta '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' 'iso-8859-1'
+meta '<meta content="text/html; charset=gb2312" http-equiv="content-type">' 'gb2312'
+meta '<meta http-equiv=Content-Type content="text/html;charset=euc-jp">' 'euc-jp'
+meta '<meta http-equiv="Content-Type" content="text/html; charset = koi8-r ">' 'koi8-r'
+meta '<meta http-equiv="Content-Type" content="text/html; charset=utf-8;">' 'utf-8'
+meta '<meta http-equiv=content-type content=text/html;charset=utf-8 />' 'utf-8'
+meta '<meta http-equiv=content-type content=text/html;charset=utf-8>' 'utf-8'
+
+# first resolvable meta wins; unrelated metas are skipped
+meta '<meta name="viewport" content="width=1"><meta charset="koi8-r">' 'koi8-r'
+meta '<meta charset="utf-8"><meta http-equiv="Content-Type" content="text/html; charset=gb2312">' 'utf-8'
+
+# duplicate attribute: first wins, as in browsers
+meta '<meta http-equiv="Content-Type" content="text/html; charset=big5" http-equiv="refresh">' 'big5'
+
+# negatives: wrong tag/attribute, empty value
+meta '<p>x</p>' '(none)'
+meta '<meta name="desc" content="charset=bogus">' '(none)'
+meta '<meta http-equiv="refresh" content="0; url=x.html">' '(none)'
+meta '<body charset="utf-8">' '(none)'
+meta '<meta charset="">' '(none)'
+
+# hostile truncations: no crash, no match
+meta '<meta charset="utf-8' '(none)'
+meta '<meta charset=' '(none)'
+meta '<meta ' '(none)'
+meta '<meta' '(none)'
+
+# hts_isStringUTF8: valid-UTF-8 probe (guards the #180 link conversion).
+utf8() {
+    test "$(httrack -O /dev/null -#test=isutf8 "$1")" == "$2" || exit 1
+}
+utf8 'plain' '1'
+utf8 'café' '1'
+utf8 '统计' '1'
+utf8 "$(printf 'caf\xe9')" '0'              # latin-1 accent
+utf8 "$(printf '\xc3')" '0'                 # truncated lead byte
+utf8 "$(printf '\xa9')" '0'                 # lone continuation byte
+utf8 "$(printf '\xf0\x9f\x98\x80')" '1'     # 4-byte emoji
+utf8 "$(printf '\xc0\xaf')" '0'             # overlong (latin-1 "À¯" must stay convertible)
+utf8 "$(printf '\xed\xa0\x80')" '0'         # UTF-16 surrogate
+utf8 "$(printf '\xf4\x90\x80\x80')" '0'     # above U+10FFFF
+utf8 "$(printf '\xf8\x88\x80\x80\x80')" '0' # legacy 5-byte form

tests/11_crawl-international.test

@@ -22,9 +22,9 @@ bash crawl-test.sh \
     httrack http://ut.httrack.com/unicode-links/utf8.html
 
 bash crawl-test.sh \
-    --errors 4 --files 7 \
-    --found ut.httrack.com/unicode-links/café3860.html \
-    --found ut.httrack.com/unicode-links/café9fa8.html \
+    --errors 2 --files 9 \
+    --found ut.httrack.com/unicode-links/café3860.html \
+    --found ut.httrack.com/unicode-links/café9fa8.html \
     --found ut.httrack.com/unicode-links/café30f4.html \
     --found ut.httrack.com/unicode-links/café5e1f.html \
     --found ut.httrack.com/unicode-links/café7b30.html \

tests/36_local-bigcrawl.test

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
-# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
+# must 304-revalidate. 361 = 1 index + 96 pages + 192 imgs + 5 shared + 61
 # family + 6 singles; the 4 planted errors write -o1 pages, not counted.
 
 set -euo pipefail
@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors 4 --files 360 \
+    --errors 4 --files 361 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \
@@ -40,6 +40,7 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --not-found 'big/x/bj.jar' \
     --not-found 'big/x/is1.png' \
     --not-found 'big/x/concat.html' \
+    --file-matches 'big/f1/gzid.html' '<p>gzid</p>' \
     --file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
     --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
     --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \

tests/41_local-utf8-link.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# #180: raw non-ASCII hrefs must reach the wire UTF-8 percent-encoded whatever
+# the declared page charset; each PDF exists only at its exact UTF-8 path.
+
+set -euo pipefail
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
+    --found 'charset/header/统计大数据服务平台.pdf' \
+    --found 'charset/meta5/统计大数据服务平台.pdf' \
+    --found 'charset/metaeq/统计大数据服务平台.pdf' \
+    --found 'charset/none/统计大数据服务平台.pdf' \
+    --found 'charset/latin1hdr/统计大数据服务平台.pdf' \
+    --found 'charset/latin1real/café.pdf' \
+    --found 'charset/metalatin1/déjà.pdf' \
+    --found 'charset/latin1ovl/À¡x.pdf' \
+    --found 'charset/priority/nuée.pdf' \
+    --found 'charset/preenc/统计大数据服务平台.pdf' \
+    --found 'charset/bom/统计大数据服务平台.pdf' \
+    httrack 'BASEURL/charset/index.html'

tests/Makefile.am

@@ -109,6 +109,7 @@ TESTS = \
 	37_local-cache-outage.test \
 	38_local-update-304.test \
 	39_local-delayed-cancel.test \
-	40_local-why.test
+	40_local-why.test \
+	41_local-utf8-link.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -374,6 +374,7 @@ def big_index(port):
         '<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
         '<a href="/big/f1/long.html?x=%s">long</a>'
         '<a href="/big/f1/gzok.html">gzok</a>'
+        '<a href="/big/f1/gzid.html">gzid</a>'
         '<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
         '<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
         '<a href="/big/e/404.html">e404</a>'
@@ -399,6 +400,7 @@ BIG_SIMPLE_PAGES = {
     "/big/f1/dir/": "dir index",
     "/big/f1/long.html": "long",
     "/big/f1/gzok.html": "gzok",
+    "/big/f1/gzid.html": "gzid",
     "/big/f1/protorel.html": "protorel",
     "/big/f1/abshost.html": "abshost",
     "/big/f5/dw.html": "dw target",
@@ -660,6 +662,97 @@ class Handler(SimpleHTTPRequestHandler):
     def route_intl_page(self):
         self.send_raw(b"<html><body>accented page</body></html>\n", "text/html")
 
+    # Raw non-ASCII href matrix (#180): each variant declares the page charset
+    # differently; the PDF exists only at its exact UTF-8 path, so a
+    # mis-decoded link 404s.
+    CHARSET_CJK = "统计大数据服务平台.pdf"
+    # variant -> (index Content-Type, <head> bytes, href bytes, pdf name)
+    CHARSET_VARIANTS = {
+        "header": ("text/html; charset=utf-8", b"", CHARSET_CJK.encode(), CHARSET_CJK),
+        "meta5": (
+            "text/html",
+            b'<meta charset="utf-8">',
+            CHARSET_CJK.encode(),
+            CHARSET_CJK,
+        ),
+        "metaeq": (
+            "text/html",
+            b'<meta http-equiv="Content-Type" content="text/html; charset=utf-8">',
+            CHARSET_CJK.encode(),
+            CHARSET_CJK,
+        ),
+        "none": ("text/html", b"", CHARSET_CJK.encode(), CHARSET_CJK),
+        "latin1hdr": (
+            "text/html; charset=iso-8859-1",
+            b"",
+            CHARSET_CJK.encode(),
+            CHARSET_CJK,
+        ),
+        # genuine latin-1 href: the charset conversion must still apply
+        "latin1real": (
+            "text/html; charset=iso-8859-1",
+            b"",
+            "café.pdf".encode("latin-1"),
+            "café.pdf",
+        ),
+        # latin-1 declared by META only: the meta parser is load-bearing
+        "metalatin1": (
+            "text/html",
+            b'<meta charset="iso-8859-1">',
+            "déjà.pdf".encode("latin-1"),
+            "déjà.pdf",
+        ),
+        # latin-1 bytes that form an overlong UTF-8 shape: strict validation
+        # must still convert them
+        "latin1ovl": (
+            "text/html; charset=iso-8859-1",
+            b"",
+            "À¡x.pdf".encode("latin-1"),
+            "À¡x.pdf",
+        ),
+        # header wins over meta: latin-1 href only resolves if iso-8859-1 is kept
+        "priority": (
+            "text/html; charset=iso-8859-1",
+            b'<meta charset="utf-8">',
+            "nuée.pdf".encode("latin-1"),
+            "nuée.pdf",
+        ),
+        "preenc": ("text/html", b"", quote(CHARSET_CJK).encode(), CHARSET_CJK),
+        "bom": ("text/html", b"", CHARSET_CJK.encode(), CHARSET_CJK),
+    }
+
+    def route_charset(self):
+        path = unquote(urlsplit(self.path).path)
+        parts = path.split("/")
+        if path == "/charset/index.html":
+            self.send_html(
+                "".join(
+                    '\t<a href="%s/index.html">%s</a>\n' % (v, v)
+                    for v in self.CHARSET_VARIANTS
+                )
+            )
+            return
+        if len(parts) == 4 and parts[2] in self.CHARSET_VARIANTS:
+            ctype, head, href, pdf = self.CHARSET_VARIANTS[parts[2]]
+            if parts[3] == "index.html":
+                body = (
+                    b"<html><head>"
+                    + head
+                    + b'</head><body><a href="'
+                    + href
+                    + b'">doc</a></body></html>'
+                )
+                if parts[2] == "bom":
+                    body = b"\xef\xbb\xbf" + body
+                self.send_raw(body, ctype)
+                return
+            if parts[3] == pdf:
+                self.send_raw(self.FAKE_PDF, "application/pdf")
+                return
+        self.send_response(404)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
     # resume / 416 loop (#206): the first GET stalls after a prefix so the crawl
     # can be interrupted (partial + temp-ref); every later request is 416.
     RESUME_PREFIX = b"PARTIAL-" + b"x" * 4096  # flushed before the stall
@@ -1079,6 +1172,13 @@ class Handler(SimpleHTTPRequestHandler):
                     "text/html",
                     extra=[("Content-Encoding", "gzip")],
                 )
+            elif path == "/big/f1/gzid.html":
+                # Plain body mislabeled as gzip: identity fallback keeps it (#47)
+                self.big_send(
+                    body,
+                    "text/html",
+                    extra=[("Content-Encoding", "gzip")],
+                )
             else:
                 self.big_send(body, "text/html")
         elif path == "/big/f1/list.html":
@@ -1194,6 +1294,9 @@ class Handler(SimpleHTTPRequestHandler):
         if path.startswith("/big/"):
             self.route_big()
             return True
+        if path.startswith("/charset/"):
+            self.route_charset()
+            return True
         # Match percent-encoded paths (accented #157 route) by their decoded form.
         handler = self.ROUTES.get(path) or self.ROUTES.get(unquote(path))
         if handler is not None: