Remove commented-out code across the engine

Sweep of ~650 lines of //-disabled statements, declarations, and calls
(plus three /*-wrapped dead blocks in htscoremain.c and htscore.c and
the MFC pApp remnant in htsserver.c). Prose comments stay, including
labels that describe live code; usage examples and config-knob docs
(HTS_TRACE_MALLOC, the htsweb.c DEBUG toggle) were kept. Insertions are
clang-format rewraps of live lines that became touched by the deletions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.gitignore

@@ -39,3 +39,6 @@ Makefile
 
 # Editor / autotools backup files.
 *~
+
+# Python bytecode (tests/local-server.py).
+__pycache__/

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.10], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,10 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:2:0: 3.49.10 only appends tail fields to the options struct (no existing
+# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
-# symbol or offset changed vs 3.49.9), so it stays soname .so.3; bump revision.
+# installed headers (no struct layout or exported signature changed vs
+# 3.49.10), so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:2:0"
+VERSION_INFO="3:3:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

debian/changelog

@@ -1,3 +1,24 @@
+httrack (3.49.11-1) unstable; urgency=medium
+
+  * New upstream release: crawl correctness and security fixes (network-facing
+    buffer overflows, file-type detection, redirect handling) and modernized
+    web defaults; full list in history.txt.
+  * Add DEP-12 upstream metadata (#466).
+  * Bump debhelper compat to 14 (#466).
+  * Drop the redundant Priority field and update the NMU lintian override to
+    the current tag names (#466).
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 05 Jul 2026 00:03:18 +0200
+
+httrack (3.49.10-2) unstable; urgency=medium
+
+  * Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
+    absent (the local-server tests need python3, which the buildds lack). Add
+    patches/skip-local-pause-test-without-python3.patch to guard the test on
+    python3 up front, like its siblings, so it skips cleanly.
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 20:18:46 +0200
+
 httrack (3.49.10-1) unstable; urgency=medium
 
   * New upstream release: new download-pacing and URL-handling options plus a

history.txt

@@ -4,6 +4,23 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-11
++ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
++ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
++ New: follow <source> and <track> media elements as embedded links (#451)
++ New: added modern web MIME types to the type/extension table (#448)
++ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
++ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
++ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
++ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
++ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
++ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
++ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
++ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
++ Changed: decode the full WHATWG set of HTML named character references (#443)
++ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
++ Changed: multiple internal hardening, build, test and CI improvements
+
 3.49-10
 + New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
 + New: --pause to space out file downloads by a random delay (#185)

src/htsalias.c

@@ -283,7 +283,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
-      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -361,7 +360,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
-              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsback.c

@@ -43,14 +43,12 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsback.h"
 
-//#ifdef _WIN32
 #include "htsftp.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
 #error HTS_USEZLIB not defined
 #endif
-//#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus
@@ -572,9 +570,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
         && back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
       if (back[p].status == STATUS_READY) {
         hts_log_print(opt, LOG_WARNING,
-                      "file not stored in cache due to bogus state (broken size, expected "
+                      "incomplete transfer (expected " LLintP
-                      LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
+                      " bytes, got " LLintP
-                      back[p].r.size, back[p].url_adr, back[p].url_fil);
+                      "): file not cached, will be retried on the next update"
+                      " (use -%%B to cache anyway): %s%s",
+                      back[p].r.totalsize, back[p].r.size, back[p].url_adr,
+                      back[p].url_fil);
       } else {
         hts_log_print(opt, LOG_INFO,
                       "incomplete file not yet stored in cache (expected "
@@ -594,7 +595,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
 #if HTS_USEZLIB
         if (back[p].r.compressed) {
           if (back[p].r.size > 0) {
-            //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
             // stats
             back[p].compressed_size = back[p].r.size;
             // en mémoire -> passage sur disque
@@ -879,11 +879,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                          back[p].url_fil, NULL);
           } else {
             /* Partial file, but marked as "ok" ? */
-            hts_log_print(opt, LOG_WARNING,
+            hts_log_print(
-                          "file not stored in cache due to bogus state (incomplete type with %s (%d), size "
+                opt, LOG_WARNING,
-                          LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
+                "file with unresolved type not cached (%s (%d), size " LLintP
-                          (LLint) back[p].r.size, back[p].url_adr,
+                "): %s%s",
-                          back[p].url_fil);
+                back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
+                back[p].url_adr, back[p].url_fil);
           }
         }
 
@@ -923,7 +924,6 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
     /* clear everything but connection: switch, close, and reswitch */
     back_connxfr(src, &tmp);
     back_delete(opt, cache, sback, p);
-    //deletehttp(src);
     back_connxfr(&tmp, src);
     src->req.flush_garbage = 1; /* ignore CRLF garbage */
     return 1;
@@ -1358,9 +1358,39 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
   return 0;
 }
 
+/* Move a still-writing .delayed placeholder to its final name (#483). */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname) {
+  hts_boolean renamed;
+
+  if (!back->r.is_write || back->tmpfile != NULL ||
+      !IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
+    return HTS_TRUE; /* nothing bound to the placeholder name */
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
+  if (renamed && (back->status == STATUS_READY ||
+                  (back->r.out = FOPEN(newname, "ab")) != NULL)) {
+    filenote(&opt->state.strc, newname, NULL);
+    hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
+                  newname);
+    return HTS_TRUE;
+  }
+  /* partial lost: drop only what we own (Windows rename won't overwrite) */
+  hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
+                back->url_sav, newname);
+  back->r.statuscode = STATUSCODE_INVALID;
+  strcpybuff(back->r.msg, "Write error on disk");
+  back->r.is_write = 0;
+  (void) UNLINK(renamed ? newname : back->url_sav);
+  return HTS_FALSE;
+}
+
 // effacer entrée
 /* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
-static void back_delayed_discard(httrackp *opt, lien_back *back) {
+void back_delayed_discard(httrackp *opt, lien_back *back) {
   if (back->r.out != NULL) {
     fclose(back->r.out);
     back->r.out = NULL;
@@ -1403,7 +1433,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].url_adr, back[p].url_fil, back[p].url_sav);
       }
       if (cache != NULL) {
-        //hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
         back_finalize(opt, cache, sback, p);
       }
     }
@@ -1544,7 +1573,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     if (back[p].r.soc != INVALID_SOCKET) {      /* we never know */
       deletehttp(&back[p].r);
     }
-    //memset(&(back[p].r), 0, sizeof(htsblk)); 
     hts_init_htsblk(&back[p].r);
     back[p].r.location = back[p].location_buffer;
 
@@ -1552,7 +1580,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     strcpybuff(back[p].url_adr, adr);
     strcpybuff(back[p].url_fil, fil);
     strcpybuff(back[p].url_sav, save);
-    //back[p].links_index = links_index;
     // copier referer si besoin
     strcpybuff(back[p].referer_adr, "");
     strcpybuff(back[p].referer_fil, "");
@@ -1791,8 +1818,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
                             back[p].url_adr, back[p].url_fil);
             }
             back[p].r.notmodified = 1;  // fichier non modifié
-            back[p].status = STATUS_READY;      // OK prêt
+            back[p].status = STATUS_READY; // OK prêt
-            //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
             back_set_finished(sback, p);
 
             // finalize transfer
@@ -1807,7 +1833,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
           } else {              // erreur
             // effacer r
             hts_init_htsblk(&back[p].r);
-            //memset(&(back[p].r), 0, sizeof(htsblk)); 
             back[p].r.location = back[p].location_buffer;
             // et continuer (chercher le fichier)
           }
@@ -2002,7 +2027,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       // ouvrir liaison, envoyer requète
       // ne pas traiter ou recevoir l'en tête immédiatement
       hts_init_htsblk(&back[p].r);
-      // memset(&(back[p].r), 0, sizeof(htsblk));
       back[p].r.location = back[p].location_buffer;
       // fresh connect: address list not yet probed, start at the first
       sback->connect_fallback[p].addr_index = 0;
@@ -2053,7 +2077,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #if HTS_USEOPENSSL
       else if (strfield(back[p].url_adr, "https://")) {     // let's rock
         back[p].r.ssl = 1;
-        // back[p].r.ssl_soc = NULL;
         back[p].r.ssl_con = NULL;
       }
 #endif
@@ -2119,9 +2142,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
         back[p].rateout = -1;   // pas de gestion (default)
       }
 
-      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
+      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
-      // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
+      // cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
-      //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
+      // page!)
 
 #if CNXDEBUG
       printf("Xfopen ok, poll..\n");
@@ -2139,7 +2162,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       if (soc == INVALID_SOCKET) {      // erreur socket
         back[p].status = STATUS_READY;  // FINI
         back_set_finished(sback, p);
-        //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
         back[p].r.soc = INVALID_SOCKET;
       } else {
         if (!back[p].r.is_file)
@@ -2160,8 +2182,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
     // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
     // le lien est considéré comme traité
-    //if (back[p].soc<0)  // erreur
-    //  return -1;
 
     return 0;
   } else {
@@ -2234,9 +2254,6 @@ void back_solve(httrackp * opt, lien_back * back) {
     } else {
       hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
     }
-    //if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
-    //  hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
-    //}
   }
 }
 
@@ -2285,11 +2302,6 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
         (void) back_flush_output(opt, cache, sback, i); // flush output buffers
         usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
                     back[i].url_fil);
-        //if (back[i].links_index >= 0) {
-        //  assertf(back[i].links_index < opt->hash->max_lien);
-        //  opt->hash->liens[back[i].links_index]->pass2 = -1;
-        //  // *back[i].pass2_ptr=-1;  // Done!
-        //}
         /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
         {
           int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME );       // lecture type 0 (sav)
@@ -2473,9 +2485,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     busy_state = busy_recv = 0;
 
-#if 0
-    check_rate(stat_timestart, opt->maxrate);   // vérifier taux de transfert
-#endif
     // inscrire les sockets actuelles, et rechercher l'ID la plus élevée
     FD_ZERO(&fds);
     FD_ZERO(&fds_c);
@@ -2485,8 +2494,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
     nfds = INVALID_SOCKET;
 
     max_c = 1;
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       // en cas de gestion du connect préemptif
@@ -2523,8 +2531,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         // poll pour la lecture sur les sockets
       if ((back[i].status > 0) && (back[i].status < 100)) {     // en réception http
 
-#if BDEBUG==1
+#if BDEBUG == 1
-        //printf("....socket in progress: %d\n",back[i].r.soc);
 #endif
         // non local et non ftp
         if (!back[i].r.is_file) {
@@ -2610,8 +2617,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       busy_recv = 0;
 
     // recevoir les données arrivées
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       if (back[i].status > 0) {
@@ -2751,7 +2757,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back[i].rateout_time = back[i].ka_time_start;
             }
             // envoyer header
-            //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
             HTS_STAT.stat_nrequests++;
             if (!back[i].head_request)
               http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2818,7 +2823,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 #if HTS_XGETHOST
       else if (back[i].status == STATUS_WAIT_DNS) {     // attendre gethostbyname
 #if DEBUGDNS
-        //printf("status 101 for %s\n",back[i].url_adr);
 #endif
 
         if (!gestion_timeout)
@@ -3091,8 +3095,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // Si réception chunk, tester si on est pas à la fin!
           if (back[i].status == 1) {
             if (back[i].is_chunk) {     // attendre prochain chunk
-              if (back[i].r.size == back[i].r.totalsize) {      // fin chunk!
+              if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
-                //printf("chunk end at %d\n",back[i].r.size);
                 back[i].status = STATUS_CHUNK_CR;       /* fetch ending CRLF */
                 if (back[i].chunk_adr != NULL) {
                   freet(back[i].chunk_adr);
@@ -3155,11 +3158,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back_finalize(opt, cache, sback, i);
             }
 
-            if (back[i].r.totalsize >= 0) {     // tester totalsize
+            if (back[i].r.totalsize >= 0) { // tester totalsize
-              //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
-                  //#if HTS_CL_IS_FATAL
                   deleteaddr(&back[i].r);
                   if (back[i].r.size < back[i].r.totalsize)
                     back[i].r.statuscode = STATUSCODE_CONNERROR;        // recatch
@@ -3168,14 +3169,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           " expected)", (LLint) back[i].r.size,
                           (LLint) back[i].r.totalsize);
                 } else {
-                  //#else
                   // Un warning suffira..
                   hts_log_print(opt, LOG_WARNING,
                                 "Incorrect length (" LLintP "!=" LLintP
-                                " expected) for %s%s", (LLint) back[i].r.size,
+                                " expected) for %s%s",
+                                (LLint) back[i].r.size,
                                 (LLint) back[i].r.totalsize, back[i].url_adr,
                                 back[i].url_fil);
-                  //#endif
                 }
               }
             }
@@ -3302,10 +3302,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          back[i].r.size);
 #endif
                   /* End */
-                  //if (back[i].status==STATUS_CHUNK_CR) {
+                  back[i].status = STATUS_READY; // fin
-                  back[i].status = STATUS_READY;        // fin  
                   back_set_finished(sback, i);
-                  //}
 
                   // finalize transfer if not temporary
                   if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3772,7 +3770,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                     back[i].url_adr, back[i].url_fil);
 
                       // finalize
-                      //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
                       if (back[i].r.statuscode > 0) {
                         hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
                         back_finalize(opt, cache, sback, i);
@@ -3782,37 +3779,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                              back[i].url_adr, back[i].url_fil);
 #endif
 
-                      //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                     } else {    // erreur
                       back[i].status = STATUS_READY;    // terminé
                       back_set_finished(sback, i);
-                      //printf("erreur cache\n");
-
                     }
 
-/********** NO - must complete the body! ********** */
-#if 0
-                  } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
-                             || (back[i].r.statuscode == 412)
-                             || (back[i].r.statuscode == 416)
-                    ) {         // Ne pas prendre le html, erreurs connues et gérées
-#if HTS_DEBUG_CLOSESOCK
-                    DEBUG_W
-                      ("back_wait(301,302,303,307,412,416..): deletehttp\n");
-#endif
-                    // Couper connexion
-                    /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
-                    back_maydeletehttp(opt, cache, sback, i);
-
-                    back[i].status = STATUS_READY;      // terminé
-                    back_set_finished(sback, i);
-                    // finalize
-                    if (back[i].r.statuscode > 0) {
-                      hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
-                      back_finalize(opt, cache, sback, i);
-                    }
-#endif
-/********** **************************** ********** */
                   }
                   // MIME type excluded by a -mime: filter: abort, don't fetch
                   // the body (#58)
@@ -3986,7 +3957,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           deletehttp(&back[i].r);
                         }
                         back[i].r.soc = INVALID_SOCKET;
-                        //back[i].r.statuscode=206;  ????????
                         back[i].r.statuscode = STATUSCODE_NON_FATAL;
                         if (strnotempty(back[i].r.msg))
                           strcpybuff(back[i].r.msg,
@@ -4013,8 +3983,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                             hts_log_print(opt, LOG_TRACE, "finalizing empty");
                             back_finalize(opt, cache, sback, i);
                           }
-                        } else if (!back[i].r.is_chunk) {       // pas de chunk
+                        } else if (!back[i].r.is_chunk) { // pas de chunk
-                          //if (back[i].r.http11!=2) {    // pas de chunk
                           back[i].is_chunk = 0;
                           back[i].status = 1;   // start body
                         } else {
@@ -4075,14 +4044,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].status < 0) {
           if (!back[i].testmode) {      // pas en test
             UNLINK(back[i].url_sav);    // éliminer fichier (endommagé)
-            //printf("&& %s\n",back[i].url_sav);
           }
         }
 #endif
 
         /* funny log for commandline users */
-        //if (!opt->quiet) {  
-        // petite animation
         if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
           if (back[i].status == STATUS_READY) {
             if (back[i].r.statuscode == HTTP_OK)
@@ -4095,18 +4061,16 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             fflush(stdout);
           }
         }
-        //}
 
       }                         // status>0
-    }                           // for
+    } // for
 
     // vérifier timeouts
     if (gestion_timeout) {
       TStamp act;
 
       act = time_local();       // temps en secondes
-      for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-        // for(i=0;i<back_max;i++) {
         unsigned int i = (i_mod + mod_random) % (back_max);
 
         if (back[i].status > 0) {       // réception/connexion/..
@@ -4134,7 +4098,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 continue;
               }
             }
-            //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
             if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
               hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
                 back[i].url_fil);

src/htsback.h

@@ -113,6 +113,12 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
+/* Discard back's on-disk .delayed placeholder and its refname. */
+void back_delayed_discard(httrackp *opt, lien_back *back);
+/* Move back's .delayed placeholder (and open stream) to newname;
+   HTS_FALSE = file lost, slot flagged in error. */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,

src/htsbasenet.h

@@ -55,8 +55,6 @@ Please visit our Website: http://www.httrack.com
 // inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
-// #include <winsock2.h>
-// #include <tpipv6.h>
 
 #endif
 

src/htsbauth.c

@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
-  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
-  //  cookie->data[0]='\0';
-
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {

src/htscache.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsbasenet.h"
 #include "htsmd5.h"
+#include <limits.h>
 #include <time.h>
 
 #include "htszlib.h"
@@ -220,23 +221,38 @@ struct cache_back_zip_entry {
 	} \
 } while(0)
 
-/* A cache (new.zip) write failed: storage is gone (disk full / dropped share),
+/* Consecutive entry write failures before the cache stream is declared dead. */
-   so the mirror is doomed too. Abort it via exit_xh, don't crash as assertf
+#define CACHE_MAX_WRITE_FAILURES 8
-   did. */
+
+/* Cache write failed: a fatal errno or a failure streak aborts the mirror
+   (exit_xh); an isolated failure only drops the current entry. */
 static void cache_zip_write_failed(httrackp *opt, cache_back *cache,
-                                   const char *what, int zErr) {
+                                   const char *what, int zErr,
-  if (!cache->zipWriteFailed) {
+                                   hts_boolean entry_open, const char *url_adr,
-    cache->zipWriteFailed = HTS_TRUE;
+                                   const char *url_fil) {
-    if (check_fatal_io_errno()) {
+  const int fatal_errno = zErr == ZIP_ERRNO && check_fatal_io_errno();
-      hts_log_print(opt, LOG_ERROR,
+
-                    "Mirror aborted: disk full or filesystem problems");
+  cache->zipWriteFailures++;
-    } else {
+  if (fatal_errno || cache->zipWriteFailures >= CACHE_MAX_WRITE_FAILURES) {
-      hts_log_print(opt, LOG_ERROR,
+    if (!cache->zipWriteFailed) {
-                    "Mirror aborted: cache write failed (%s): %s", what,
+      cache->zipWriteFailed = HTS_TRUE;
-                    hts_get_zerror(zErr));
+      if (fatal_errno) {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: disk full or filesystem problems");
+      } else {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: cache write failed (%s): %s", what,
+                      hts_get_zerror(zErr));
+      }
     }
+    opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
+  } else {
+    if (entry_open)
+      zipCloseFileInZip((zipFile) cache->zipOutput); /* abandon, best-effort */
+    hts_log_print(opt, LOG_WARNING,
+                  "cache write failed (%s: %s), entry not cached: %s%s", what,
+                  hts_get_zerror(zErr), url_adr, url_fil);
   }
-  opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
 }
 
 /* Ajout d'un fichier en cache */
@@ -249,8 +265,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   char BIGSTK headers[8192];
   int headersSize = 0;
 
-  //int entryBodySize = 0;
-  //int entryFilenameSize = 0;
   zip_fileinfo fi;
   const char *url_save_suffix = url_save;
   int zErr;
@@ -286,10 +300,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   if (r->size < 0)              // error
     return;
 
-  // data in cache
+  // data in cache: the body must fit the 32-bit zip write API
-  if (dataincache) {
+  if (dataincache && (LLint) (int) r->size != r->size) {
-    assertf(((int) r->size) == r->size);
+    if (r->is_write && url_save != NULL && strnotempty(url_save)) {
-    //entryBodySize = (int) r->size;
+      hts_log_print(opt, LOG_WARNING,
+                    "file too large for the cache, storing headers only: %s%s",
+                    url_adr, url_fil);
+      dataincache = 0;
+    } else {
+      hts_log_print(opt, LOG_WARNING,
+                    "entry too large for the cache, not cached: %s%s", url_adr,
+                    url_fil);
+      return;
+    }
   }
 
   /* Fields */
@@ -334,8 +357,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil);     // Original URI filename
   ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix);    // Original save filename
 
-  //entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil));
-
   /* Filename */
   if (!link_has_authority(url_adr)) {
     strcpybuff(filename, "http://");
@@ -369,7 +390,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
                                    */
                                   headers, (uInt) strlen(headers), NULL, 0, NULL,       /* comment */
                                   Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
-    cache_zip_write_failed(opt, cache, "opening a cache entry", zErr);
+    cache_zip_write_failed(opt, cache, "opening a cache entry", zErr, HTS_FALSE,
+                           url_adr, url_fil);
     return;
   }
 
@@ -380,7 +402,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
         if ((zErr =
              zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr,
                                  (int) r->size)) != Z_OK) {
-          cache_zip_write_failed(opt, cache, "writing to the cache", zErr);
+          cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
+                                 HTS_TRUE, url_adr, url_fil);
           return;
         }
       }
@@ -402,8 +425,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
               if ((zErr =
                    zipWriteInFileInZip((zipFile) cache->zipOutput, buff,
                                        (int) nl)) != Z_OK) {
-                cache_zip_write_failed(opt, cache, "writing to the cache",
+                cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
-                                       zErr);
+                                       HTS_TRUE, url_adr, url_fil);
                 fclose(fp);
                 return;
               }
@@ -419,15 +442,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
 
   /* Close */
   if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) {
-    cache_zip_write_failed(opt, cache, "closing a cache entry", zErr);
+    cache_zip_write_failed(opt, cache, "closing a cache entry", zErr, HTS_FALSE,
+                           url_adr, url_fil);
     return;
   }
 
   /* Flush */
   if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) {
-    cache_zip_write_failed(opt, cache, "flushing the cache", zErr);
+    cache_zip_write_failed(opt, cache, "flushing the cache", zErr, HTS_FALSE,
+                           url_adr, url_fil);
     return;
   }
+
+  cache->zipWriteFailures = 0; /* entry stored: reset the failure streak */
 }
 
 #else
@@ -694,7 +721,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
         char BIGSTK headerBuff[8192 + 2];
         int readSizeHeader;
 
-        //int totalHeader = 0;
         int dataincache = 0;
 
         /* For BIG comments */
@@ -745,13 +771,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                                    HTS_URLMAXSIZE * 2);
               ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo,
                                    sizeof(r.cdispo));
-              //ZIP_READFIELD_STRING(line, value, "X-Addr", ..);            // Original address
-              //ZIP_READFIELD_STRING(line, value, "X-Fil", ..);            // Original URI filename
               ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
                                    sizeof(previous_save_));
             }
-          } while(offset < readSizeHeader && !lineEof);
+          } while (offset < readSizeHeader && !lineEof);
-          //totalHeader = offset;
 
           /* Previous entry */
           if (previous_save_[0] != '\0') {
@@ -768,6 +791,15 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
           }
 
+          /* A negative X-Size is corrupt; so is one >= INT_MAX when the data
+             is in the zip (the write path asserts int-sized). Headers-only
+             entries legitimately exceed INT_MAX (>2GB body on disk): keep
+             them, or every update would re-fetch the file. */
+          if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
+            r.statuscode = STATUSCODE_INVALID;
+            strcpybuff(r.msg, "Cache Read Error : Bad Size");
+          }
+
           /* Complete fields */
           r.totalsize = r.size;
           r.adr = NULL;
@@ -794,7 +826,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             }                   // otherwise, the ZIP file is supposed to be consistent with data.
           }
           /* Read data ? */
-          else {                /* ne pas lire uniquement header */
+          else if (r.statuscode !=
+                   STATUSCODE_INVALID) { /* ne pas lire uniquement header */
             int ok = 0;
 
 #if HTS_DIRECTDISK
@@ -923,7 +956,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg,
                              "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                 }
               }
 
@@ -958,7 +990,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg,
                                "Previous cache file not found (empty filename)");
                   }
-                } else {        /* Read in memory from disk */
+                } else if (r.size >= INT_MAX) { /* too big to read in memory */
+                  r.statuscode = STATUSCODE_INVALID;
+                  strcpybuff(r.msg, "Cache Read Error : Bad Size");
+                } else { /* Read in memory from disk */
                   FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
 
                   if (fp != NULL) {
@@ -998,7 +1033,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg, "Cache Read Error : Read Data");
                   } else
                     *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                 } else {        // erreur
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg, "Cache Memory Error");
@@ -1177,17 +1211,14 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
         // sécurité
         r.adr = NULL;
         r.out = NULL;
-        ////r.location=NULL;  non, fixée lors des 301 ou 302
         r.fp = NULL;
 
         if ((r.statuscode >= 0) && (r.statuscode <= 999)
             && (r.notmodified >= 0) && (r.notmodified <= 9)) {  // petite vérif intégrité
-          if ((save) && (!header_only)) {       /* ne pas lire uniquement header */
+          if ((save) && (!header_only)) { /* ne pas lire uniquement header */
-            //int to_file=0;
 
             r.adr = NULL;
             r.soc = INVALID_SOCKET;
-            // // r.location=NULL;
 
 #if HTS_DIRECTDISK
             // Court-circuit:
@@ -1196,12 +1227,11 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
               int ok = 0;
 
               r.is_write = 1;   // écrire
-              if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {  // un fichier existe déja
+              if (fexist_utf8(fconv(catbuff, sizeof(catbuff),
-                //if (fsize_utf8(fconv(save))==r.size) {  // même taille -- NON tant pis (taille mal declaree)
+                                    save))) { // un fichier existe déja
                 ok = 1;         // plus rien à faire
                 filenote(&opt->state.strc, save, NULL); // noter comme connu
                 file_notify(opt, adr, fil, save, 0, 0, 0);
-                //}
               }
 
               if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur!
@@ -1252,7 +1282,6 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg,
                              "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                 }
               }
 
@@ -1300,14 +1329,13 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg, "Cache Read Error : Read Data");
                   } else
                     *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                 } else {        // erreur
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg, "Cache Memory Error");
                 }
               }
             }
-          }                     // si save==null, ne rien charger (juste en tête)
+          } // si save==null, ne rien charger (juste en tête)
         } else {
 #if DEBUGCA
           printf("Cache Read Error : Bad Data");
@@ -1420,6 +1448,86 @@ static int hts_rename(httrackp * opt, const char *a, const char *b) {
   return rename(a, b);
 }
 
+/* Pathname of a file inside the mirror dir (rotating concat buffer). */
+static char *reconcile_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+/* Interrupted-run heuristic: prefer the old generation when the new cache
+   stalled below NEW_TINY while the old one grew past OLD_SOLID (historical
+   arbitrary thresholds). */
+#define CACHE_RECONCILE_NEW_TINY 32768
+#define CACHE_RECONCILE_OLD_SOLID 65536
+
+/* Replace the new-generation file by the old one, when the old one exists. */
+static void reconcile_promote(httrackp *opt, const char *oldname,
+                              const char *newname) {
+  if (fexist(reconcile_path(opt, oldname))) {
+    remove(reconcile_path(opt, newname));
+    rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
+  }
+}
+
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
+  switch (mode) {
+  case CACHE_RECONCILE_PROMOTE:
+    /* Previous run rotated new.* to old.* then died before writing: promote
+       the old generation back, whichever format it uses. */
+    if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if ((!fexist(reconcile_path(opt, "hts-cache/new.dat")) ||
+         !fexist(reconcile_path(opt, "hts-cache/new.ndx"))) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_INTERRUPTED:
+    /* Aborted run: keep the larger generation when the new cache is
+       suspiciously small next to the old one. The new file must exist: fsize()
+       is -1 for a missing file, which would spuriously pass the "< TINY" test
+       and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
+     */
+    if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
+      break;
+    if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.zip")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            fsize(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/new.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.dat")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            fsize(reconcile_path(opt, "hts-cache/new.dat"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_ROLLBACK:
+    /* Nothing transferred: restore the previous generation and sidecars. */
+    reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    reconcile_promote(opt, "hts-cache/old.lst", "hts-cache/new.lst");
+    reconcile_promote(opt, "hts-cache/old.txt", "hts-cache/new.txt");
+    break;
+  }
+}
+
 // renvoyer uniquement en tête, ou NULL si erreur
 // return NULL upon error, and set -1 to r.statuscode
 htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
@@ -1450,25 +1558,10 @@ void cache_init(cache_back * cache, httrackp * opt) {
       mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
             HTS_PROTECT_FOLDER);
 #endif
-      if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) {       // il existe déja un cache précédent.. renommer
+      if ((fexist(fconcat(
-        /* Previous cache from the previous cache version */
+              OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-#if 0
+              StringBuff(opt->path_log),
-        /* No.. reuse with old httrack releases! */
+              "hts-cache/new.zip")))) { // a previous cache exists.. rename it
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.dat"));
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.ndx"));
-#endif
         /* Previous cache version */
         if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {     // il existe déja un cache précédent.. renommer
           rename(fconcat
@@ -1509,7 +1602,13 @@ void cache_init(cache_back * cache, httrackp * opt) {
         } else {
           hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
         }
-      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {        // il existe déja un cache précédent.. renommer
+      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.dat"))) &&
+                 (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.ndx")))) { // a previous cache
+                                                           // exists.. rename it
 #if DEBUGCA
         printf("work with former cache\n");
 #endif
@@ -1538,7 +1637,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
                 "hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                                               StringBuff(opt->path_log),
                                               "hts-cache/old.ndx"));
-      } else {                  // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel
+      } else { // one or both cache files missing: remove the remaining one
 #if DEBUGCA
         printf("new cache\n");
 #endif
@@ -1907,12 +2006,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                    "hts-cache/new.lst"), "wb");
           strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
           opt->state.strc.lst = cache->lst;
-          //{
-          //filecreate_params tmp;
-          //strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //tmp.lst=cache->lst;                 // fichier lst
-          //filenote("",&tmp);        // initialiser filecreate
-          //}
 
           // supprimer old.txt
           if (fexist
@@ -2001,12 +2094,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                    "hts-cache/new.lst"), "wb");
           strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
           opt->state.strc.lst = cache->lst;
-          //{
-          //  filecreate_params tmp;
-          //  strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //  tmp.lst=cache->lst;                 // fichier lst
-          //  filenote("",&tmp);        // initialiser filecreate
-          //}
 
           // supprimer old.txt
           if (fexist
@@ -2038,8 +2125,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                     "statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"
                     LF);
           }
-          // test
-          // cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9);
         }                       // cache->ndx!=NULL
       }                         //cache->zipOutput != NULL
 

src/htscache.h

@@ -78,6 +78,17 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
+/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
+typedef enum {
+  CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */
+  CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
+  CACHE_RECONCILE_ROLLBACK     /* nothing transferred: restore the old one */
+} hts_cache_reconcile_mode;
+
+/* Reconcile the on-disk cache generations according to mode; a no-op when
+   the involved files are absent. */
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
+
 int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
                     const char *str2, char *outbuff, int len);
 int cache_readdata(cache_back * cache, const char *str1, const char *str2,

src/htscache_selftest.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 #include "htszlib.h"
 
 #include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -321,6 +322,7 @@ typedef struct {
   size_t budget;  /**< bytes allowed through before writes start failing */
   int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
   int writes;     /**< zwrite call count, to detect re-entry into the stream */
+  int fail_once;  /**< recover (unlimited budget) after the first failure */
 } writefail_inject;
 
 /* zwrite that copies until the budget runs out, then fails with inj->fail_errno
@@ -335,6 +337,8 @@ static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
     inj->budget -= (size_t) size;
     return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
   }
+  if (inj->fail_once)
+    inj->budget = (size_t) -1; /* the backend recovers after this failure */
   errno = inj->fail_errno;
   return 0; /* short write -> the minizip op returns an error */
 }
@@ -373,9 +377,50 @@ static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
   freet(bodycopy);
 }
 
-/* #174/#219: a failing cache write used to crash via assertf(); it must instead
+/* Store an entry claiming a >2GB body; the degrade path never reads data. */
-   stop the mirror (exit_xh = -1) without crashing. Assert that, plus the cache
+static void writefail_store_oversized(httrackp *opt, cache_back *cache,
-   is flagged and a sibling write doesn't re-enter the broken stream. */
+                                      const char *fil, int is_write) {
+  htsblk r;
+  char locbuf[4];
+
+  hts_init_htsblk(&r);
+  r.statuscode = 200;
+  r.size = (LLint) INT_MAX + 1;
+  strcpybuff(r.msg, "OK");
+  strcpybuff(r.contenttype, "application/octet-stream");
+  locbuf[0] = '\0';
+  r.location = locbuf;
+  r.is_write = (short int) is_write;
+  cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
+}
+
+/* Read back `entryname`: extra field (cached headers) and body. Returns the
+   body length, or -1 if the entry is absent or unreadable. */
+static int writefail_read_entry(const char *path, const char *entryname,
+                                char *extra, size_t extralen, char *body,
+                                size_t bodylen) {
+  unzFile z = unzOpen(path);
+  int n = -1;
+
+  if (z == NULL)
+    return -1;
+  if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
+      unzOpenCurrentFile(z) == UNZ_OK) {
+    const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
+
+    if (elen >= 0) {
+      extra[elen] = '\0';
+      n = unzReadCurrentFile(z, body, (unsigned) bodylen);
+    }
+    unzCloseCurrentFile(z);
+  }
+  unzClose(z);
+  return n;
+}
+
+/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
+   stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
+ */
 int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   int fail = 0;
   char path[HTS_URLMAXSIZE];
@@ -388,9 +433,8 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   gen_body(body, body_len, 1 /* incompressible */);
   fconcat(path, sizeof(path), dir, "/wfail.zip");
 
-  /* phase 0: fail on the body write, fatal errno (ENOSPC, the disk-full
+  /* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
-     branch). phase 1: fail on the open, non-fatal errno (EIO, dropped-share
+     drops entries until the streak caps out, then aborts. */
-     branch). Both must abort the mirror. */
   for (phase = 0; phase < 2; phase++) {
     cache_back cache;
     writefail_inject inj;
@@ -399,6 +443,7 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     inj.budget = (phase == 0) ? 4096 : 0;
     inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
     inj.writes = 0;
+    inj.fail_once = 0;
     memset(&cache, 0, sizeof(cache));
     cache.type = 1;
     cache.log = stderr;
@@ -412,7 +457,25 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
 
     opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
-    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (phase == 0) {
+      writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    } else {
+      /* the abort must land exactly on the 8th consecutive failure */
+      int i;
+
+      for (i = 0; i < 7; i++) {
+        char fil[32];
+
+        snprintf(fil, sizeof(fil), "/b%d.bin", i);
+        writefail_store(opt, &cache, fil, body, 16);
+      }
+      if (cache.zipWriteFailed) {
+        fprintf(stderr, "cache-writefail: phase 1: aborted before the "
+                        "8th consecutive failure\n");
+        fail++;
+      }
+      writefail_store(opt, &cache, "/b7.bin", body, 16);
+    }
     if (!cache.zipWriteFailed) {
       fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
               phase);
@@ -443,6 +506,136 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
   }
 
+  /* failures with successes in between reset the streak: never aborts */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    int i;
+
+    inj.budget = (size_t) -1;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    for (i = 0; i < 10; i++) {
+      char fil[32];
+
+      inj.budget = 0; /* this store fails */
+      snprintf(fil, sizeof(fil), "/s%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+      inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
+      snprintf(fil, sizeof(fil), "/ok%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+    }
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: scattered: non-consecutive failures aborted "
+              "the mirror (flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+  }
+
+  /* isolated failure: only that entry drops; a later sibling round-trips */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = 4096;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 1;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: isolated failure aborted the mirror "
+              "(flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    writefail_store(opt, &cache, "/blob2.bin", body, 16);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+    n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 16 || memcmp(rbody, body, 16) != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: sibling entry lost after a skipped "
+              "entry (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
+  /* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = (size_t) -1; /* no injected failure */
+    inj.fail_errno = 0;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
+    writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: mirror aborted (flagged=%d, "
+              "exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody)) >= 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: in-memory entry was stored\n");
+      fail++;
+    }
+    n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
+      fprintf(stderr,
+              "cache-writefail: oversize: on-disk entry not stored "
+              "headers-only (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
   freet(body);
   return fail;
 }
@@ -716,3 +909,494 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
 
   return failures;
 }
+
+/* --- hts_cache_reconcile() policies -------------------------------------- */
+
+/* All reconcile inputs/outputs, wiped between cases. */
+static const char *const reconcile_files[] = {
+    "hts-cache/new.zip", "hts-cache/old.zip",   "hts-cache/new.dat",
+    "hts-cache/old.dat", "hts-cache/new.ndx",   "hts-cache/old.ndx",
+    "hts-cache/new.lst", "hts-cache/old.lst",   "hts-cache/new.txt",
+    "hts-cache/old.txt", "hts-in_progress.lock"};
+
+static char *reconcile_st_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+static void reconcile_wipe(httrackp *opt) {
+  size_t i;
+
+  for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
+    remove(reconcile_st_path(opt, reconcile_files[i]));
+}
+
+/* Create a filler file of exactly `size` bytes. */
+static void reconcile_put(httrackp *opt, const char *name, size_t size) {
+  FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
+  static const char filler[1024] = {'x'};
+
+  assertf(fp != NULL);
+  while (size > 0) {
+    const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
+
+    assertf(fwrite(filler, 1, n, fp) == n);
+    size -= n;
+  }
+  fclose(fp);
+}
+
+/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
+static int reconcile_expect(httrackp *opt, const char *name, off_t size,
+                            const char *what) {
+  const off_t got = fsize(reconcile_st_path(opt, name));
+
+  if (got != size) {
+    fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
+            name, (int) got, (int) size);
+    return 1;
+  }
+  return 0;
+}
+
+int cache_reconcile_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  /* around the interrupted-run thresholds (new < 32768, old > 65536) */
+  static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
+
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* PROMOTE: a zip old generation replaces a missing new one */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
+
+  /* PROMOTE: an existing new.zip is left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
+
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
+
+  /* INTERRUPTED: no lock file, no action */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
+
+  /* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
+     spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
+
+  /* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
+     zip caches: the arm was gated on a legacy new.dat) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
+
+  /* INTERRUPTED: old below the confidence threshold, keep new */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
+
+  /* INTERRUPTED: new big enough to trust, keep it */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", MID);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
+
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
+
+  /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
+     its only good generation here) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
+
+  /* ROLLBACK: sidecars are restored regardless of format */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.lst", TINY);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
+
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
+
+  /* ROLLBACK: nothing to restore, the new generation stays */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
+/* --- read-side corruption injection --------------------------------------- */
+
+/* canary read back intact after each corruption; victim gets the byte surgery
+ */
+#define CORRUPT_ADR "corrupt.example.com"
+static char corrupt_body_a[33 + 1];
+static char corrupt_body_b[44 + 1];
+
+/* Write a fresh two-entry cache: /canary.html then /victim.html. */
+static void corrupt_build(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
+              strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
+   is later overwritten with a forged oversized X-Size (same byte length). */
+static void corrupt_build_etag(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
+              corrupt_body_b, strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
+   body on disk): the shape every non-html file is stored with. */
+static void corrupt_build_disk(httrackp *opt) {
+  cache_back cache;
+  htsblk w;
+  char locw[4];
+  char BIGSTK save[HTS_URLMAXSIZE * 2];
+  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
+  char *path;
+  FILE *fp;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+          CORRUPT_ADR "/victim.bin");
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  hts_init_htsblk(&w);
+  w.statuscode = 200;
+  w.size = (LLint) sizeof(corrupt_body_b) - 1;
+  strcpybuff(w.msg, "OK");
+  strcpybuff(w.contenttype, "application/octet-stream");
+  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
+  locw[0] = '\0';
+  w.location = locw;
+  w.is_write = 0;
+  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
+            0 /* all_in_cache */, NULL);
+  selftest_close(&cache);
+  /* the reader only checks this file exists; it never reads it here */
+  path = fconv(catbuff, sizeof(catbuff), save);
+  (void) structcheck(path);
+  fp = FOPEN(path, "wb");
+  assertf(fp != NULL);
+  fclose(fp);
+}
+
+/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
+static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
+                          const char *rep, size_t nth, size_t total) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, at = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + patlen <= n; k++) {
+    if (memcmp(data + k, pat, patlen) == 0) {
+      hits++;
+      if (hits == nth)
+        at = k;
+    }
+  }
+  assertf(hits == total);
+  memcpy(data + at, rep, patlen);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Garbage the first bytes of the victim's deflated data (2nd local header). */
+static void corrupt_victim_body(httrackp *opt) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, off = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + 4 <= n; k++) {
+    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
+      const size_t namelen =
+          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
+      const size_t extralen =
+          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
+
+      off = k + 30 + namelen + extralen;
+    }
+  }
+  assertf(hits == 2);
+  assertf(off != 0 && off + 4 <= n);
+  memset(data + off, 0xFF, 4);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Read the corrupt /victim.html and, in the SAME read session, the intact
+   /canary.html: the victim must be rejected (wantmsg pins which path) and the
+   canary must still decode byte-exact, proving one bad entry never taints a
+   sibling read. */
+static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
+                                     const char *wantmsg, const char *what) {
+  cache_back cache;
+  htsblk v, c;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  char BIGSTK lc[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = lc[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
+  if (v.statuscode != STATUSCODE_INVALID) {
+    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
+            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
+    fail++;
+  }
+  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
+    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
+            selftest_tag, what, v.msg, wantmsg);
+    fail++;
+  }
+  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
+  if (c.statuscode != 200 || c.adr == NULL ||
+      c.size != (LLint) strlen(corrupt_body_a) ||
+      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
+    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
+            c.statuscode);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  if (c.adr != NULL)
+    freet(c.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
+                                 const char *what) {
+  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
+}
+
+/* Headers-only probe of the disk victim: must parse OK with the size kept. */
+static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
+                                      const char *what) {
+  cache_back cache;
+  htsblk v;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
+  if (v.statuscode != 200 || v.size != wantsize) {
+    fprintf(stderr,
+            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
+            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+/* One zip corruption case: build, patch, then check victim+canary in-session.
+ */
+static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
+                            size_t nth, size_t total, const char *wantmsg,
+                            const char *what) {
+  corrupt_build(opt);
+  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
+  return corrupt_expect_victim(opt, wantmsg, what);
+}
+
+int cache_corruption_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  selftest_tag = "cache-corrupt";
+  golden_setup(opt, dir);
+
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
+                       "Cache Read Error : Read Data", "oversized X-Size");
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
+                       "Cache Read Error : Bad Size", "negative X-Size");
+  /* both entries carry the line; the victim's is the second */
+  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
+                               "Previous cache file not found (empty filename)",
+                               "blanked X-In-Cache");
+  /* smashed local file header: the entry is dropped at index load */
+  failures +=
+      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
+                       "File Cache Entry Not Found", "smashed local header");
+
+  corrupt_build(opt);
+  corrupt_victim_body(opt);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
+                                    "garbled deflate stream");
+
+  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
+     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
+     cache_add asserts size fits an int, so such a value only reaches the reader
+     from a corrupt/foreign cache; inject it by overwriting the victim's long
+     Etag line with a same-length forged X-Size line (the parser keeps the last
+     X-Size it sees), keeping the zip byte-length and offsets intact. */
+  corrupt_build_etag(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
+                                    "X-Size above INT_MAX");
+
+  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
+     is how every >2GB non-html file is stored. It must survive a header probe
+     (or every update re-fetches the file); an in-memory read still rejects. */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
+                                         "headers-only X-Size above INT_MAX");
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size above INT_MAX");
+
+  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483647AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size at INT_MAX");
+
+  /* the negative check must stay global, headers-only included */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: -2147483648AAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "headers-only negative X-Size");
+
+  return failures;
+}

src/htscache_selftest.h

@@ -52,10 +52,19 @@ int cache_selftests(httrackp *opt, const char *dir);
    committed file, never by the test). Returns the failed-check count. */
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
 
-/* #174/#219: assert a failing cache write aborts the mirror cleanly instead of
+/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
-   crashing. Returns the failed-check count. */
+   drop just the entry otherwise. Returns the failed-check count. */
 int cache_write_failure_selftest(httrackp *opt, const char *dir);
 
+/* Exercise the hts_cache_reconcile() generation policies on file fixtures
+   under <dir>. Returns the failed-check count. */
+int cache_reconcile_selftest(httrackp *opt, const char *dir);
+
+/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
+   under <dir> and assert every case degrades to STATUSCODE_INVALID without
+   tainting a sibling entry. */
+int cache_corruption_selftest(httrackp *opt, const char *dir);
+
 #endif
 
 #endif

src/htscatchurl.c

@@ -197,7 +197,6 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
-            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -207,8 +206,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
-            //strcatbuff(data,"\r\n");
+            // juste au dessus
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -841,71 +841,6 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
-/* Sample. */
-#if 0
-
-int main(int argc, char **argv) {
-  int i;
-  int hex = 0;
-  
-#define READ_INT(DEST)                                  \
-  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
-    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
-  
-  for(i = 1 ; i < argc ; i++) {
-    unsigned int uc, from, to;
-    
-    if (strcmp(argv[i], "--hex") == 0) {
-      hex = 1;
-    }
-    else if (strcmp(argv[i], "--decimal") == 0) {
-      hex = 0;
-    }
-    else if (strcmp(argv[i], "--decode") == 0) {
-#define RD fgetc_unlocked(stdin)
-#define WR(C) do {                              \
-        if (C != -1) {                          \
-          printf("%04x\n", C);                  \
-        } else if (!feof(stdin)) {              \
-          fprintf(stderr, "read error\n");      \
-          exit(EXIT_FAILURE);                   \
-        }                                       \
-      } while(0)
-      while(!feof(stdin)) {
-        READ_UNICODE(RD, WR);
-      }
-#undef RD
-#undef WR
-    }
-    else if (strcmp(argv[i], "-range") == 0
-             && i + 2 < argc
-             && (++i, 1)
-             && READ_INT(from)
-             && (++i, 1)
-             && READ_INT(to)
-             ) {
-      unsigned int i;
-      for(i = from ; i < to ; i++) {
-#define EM(C) fputc_unlocked(C, stdout)
-        EMIT_UNICODE(i, EM);
-#undef EM
-      }
-    }
-    else if (READ_INT(uc)) {
-#define EM(C) fputc_unlocked(C, stdout)
-      EMIT_UNICODE(uc, EM);
-#undef EM
-    }
-    else {
-      return EXIT_FAILURE;
-    }
-  }
-  
-  return EXIT_SUCCESS;
-}
-
-#endif
-
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA

src/htsconfig.h

@@ -54,10 +54,6 @@ Please visit our Website: http://www.httrack.com
 // extension par défaut pour fichiers n'en ayant pas
 #define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
-// #define DEFAULT_BIN_EXT   ".bin"
-// #define DEFAULT_BIN_EXT_SHORT ".bin"
-// #define DEFAULT_EXT       ".txt"
-// #define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -80,10 +76,6 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// gérer une table de hachage?
-// REMOVED
-// #define HTS_HASH 1
-
 // fast cache (build hash table)
 #define HTS_FAST_CACHE 1
 
@@ -97,27 +89,10 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
-// case-sensitive pour les dossiers et fichiers (0/1)
-// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
-// pas très utile..
-// ..et pas bcp respecté]
-// REMOVED
-// #define HTS_CASSE 0
-
-// Un fichier ayant une taille différente du content-length doit il être annulé?
-// SEE opt.tolerant and opt.http10
-// #define HTS_CL_IS_FATAL 0
-
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
-// en cas de Range: xx- donnant un Content-length: xx
-// alors skipper le fichier, considéré comme transmis
-// #define HTS_SKIP_FULL_RANGE 1
-
-// nombre max de filtres que l'utilisateur peut fixer
-// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.c

@@ -119,46 +119,93 @@ hts_log_print(opt, LOG_INFO, "engine: end"); \
 RUN_CALLBACK0(opt, end); \
 }
 
-#define XH_extuninit do { \
+#define XH_extuninit                                                           \
-  HTMLCHECK_UNINIT \
+  do {                                                                         \
-  hts_record_free(opt); \
+    HTMLCHECK_UNINIT                                                           \
-  if (filters && filters[0]) { \
+    hts_record_free(opt);                                                      \
-  freet(filters[0]); filters[0]=NULL; \
+    if (filters && filters[0]) {                                               \
-  } \
+      freet(filters[0]);                                                       \
-  if (filters) { \
+      filters[0] = NULL;                                                       \
-  freet(filters); filters=NULL; \
+    }                                                                          \
-  } \
+    if (filters) {                                                             \
-  back_delete_all(opt,&cache,sback); \
+      freet(filters);                                                          \
-  back_free(&sback); \
+      filters = NULL;                                                          \
-  checkrobots_free(&robots);\
+    }                                                                          \
-  if (cache.use) { freet(cache.use); cache.use=NULL; } \
+    back_delete_all(opt, &cache, sback);                                       \
-  if (cache.dat) { fclose(cache.dat); cache.dat=NULL; }  \
+    back_free(&sback);                                                         \
-  if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
+    checkrobots_free(&robots);                                                 \
-  if (cache.zipOutput) { \
+    if (cache.use) {                                                           \
-    zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
+      freet(cache.use);                                                        \
-    cache.zipOutput = NULL; \
+      cache.use = NULL;                                                        \
-  } \
+    }                                                                          \
-  if (cache.zipInput) { \
+    if (cache.dat) {                                                           \
-    unzClose(cache.zipInput); \
+      fclose(cache.dat);                                                       \
-    cache.zipInput = NULL; \
+      cache.dat = NULL;                                                        \
-  } \
+    }                                                                          \
-  if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
+    if (cache.ndx) {                                                           \
-  if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
+      fclose(cache.ndx);                                                       \
-  if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
+      cache.ndx = NULL;                                                        \
-  if (opt->log != NULL) fflush(opt->log); \
+    }                                                                          \
-  if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
+    if (cache.zipOutput) {                                                     \
-  if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
+      zipClose(cache.zipOutput,                                                \
-  if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
+               "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
-  if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
+      cache.zipOutput = NULL;                                                  \
-  if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
+    }                                                                          \
-  if (cache_tests)     { coucal_delete(&cache_tests); } \
+    if (cache.zipInput) {                                                      \
-  if (template_header) { freet(template_header); template_header=NULL; } \
+      unzClose(cache.zipInput);                                                \
-  if (template_body)   { freet(template_body); template_body=NULL; } \
+      cache.zipInput = NULL;                                                   \
-  if (template_footer) { freet(template_footer); template_footer=NULL; } \
+    }                                                                          \
-  hash_free(&hash); \
+    if (cache.olddat) {                                                        \
-  clearCallbacks(&opt->state.callbacks); \
+      fclose(cache.olddat);                                                    \
-  /*structcheck_init(-1);*/ \
+      cache.olddat = NULL;                                                     \
-} while(0)
+    }                                                                          \
+    if (cache.lst) {                                                           \
+      fclose(cache.lst);                                                       \
+      cache.lst = opt->state.strc.lst = NULL;                                  \
+    }                                                                          \
+    if (cache.txt) {                                                           \
+      fclose(cache.txt);                                                       \
+      cache.txt = NULL;                                                        \
+    }                                                                          \
+    if (opt->log != NULL)                                                      \
+      fflush(opt->log);                                                        \
+    if (makestat_fp) {                                                         \
+      fclose(makestat_fp);                                                     \
+      makestat_fp = NULL;                                                      \
+    }                                                                          \
+    if (maketrack_fp) {                                                        \
+      fclose(maketrack_fp);                                                    \
+      maketrack_fp = NULL;                                                     \
+    }                                                                          \
+    if (opt->accept_cookie)                                                    \
+      cookie_save(opt->cookie,                                                 \
+                  fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),           \
+                          StringBuff(opt->path_log), "cookies.txt"));          \
+    if (makeindex_fp) {                                                        \
+      fclose(makeindex_fp);                                                    \
+      makeindex_fp = NULL;                                                     \
+    }                                                                          \
+    if (cache_hashtable) {                                                     \
+      coucal_delete(&cache_hashtable);                                         \
+    }                                                                          \
+    if (cache_tests) {                                                         \
+      coucal_delete(&cache_tests);                                             \
+    }                                                                          \
+    if (template_header) {                                                     \
+      freet(template_header);                                                  \
+      template_header = NULL;                                                  \
+    }                                                                          \
+    if (template_body) {                                                       \
+      freet(template_body);                                                    \
+      template_body = NULL;                                                    \
+    }                                                                          \
+    if (template_footer) {                                                     \
+      freet(template_footer);                                                  \
+      template_footer = NULL;                                                  \
+    }                                                                          \
+    hash_free(&hash);                                                          \
+    clearCallbacks(&opt->state.callbacks);                                     \
+  } while (0)
 #define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)
 
 struct lien_buffers {
@@ -523,7 +570,6 @@ int httpmirror(char *url1, httrackp * opt) {
   hash_struct *const hashptr = &hash;
   t_cookie BIGSTK cookie;       // gestion des cookies
 
-  //char* tab_alloc=NULL;
   int ptr;                      // pointeur actuel sur les liens
 
   //
@@ -534,15 +580,12 @@ int httpmirror(char *url1, httrackp * opt) {
   // pour les stats, nombre de fichiers & octets écrits
   LLint stat_fragment = 0;      // pour la fragmentation
 
-  //TStamp istat_timestart;   // départ pour calcul instantanné
-  //
   TStamp last_info_shell = 0;
   int info_shell = 0;
 
   // filtres
   char **filters = NULL;
 
-  //int filter_max=0;
   int filptr = 0;
 
   //
@@ -582,7 +625,6 @@ int httpmirror(char *url1, httrackp * opt) {
   // noter heure actuelle de départ en secondes
   memset(&HTS_STAT, 0, sizeof(HTS_STAT));
   HTS_STAT.stat_timestart = time_local();
-  //istat_timestart=stat_timestart;
   HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
   /* reset stats */
   HTS_STAT.HTS_TOTAL_RECV = 0;
@@ -616,9 +658,6 @@ int httpmirror(char *url1, httrackp * opt) {
   // initialiser usercommand
   usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");
 
-  // initialiser structcheck
-  // structcheck_init(1);
-
   // initialiser verif_backblue
   verif_backblue(opt, NULL);
   verif_external(opt, 0, 0);
@@ -638,9 +677,6 @@ int httpmirror(char *url1, httrackp * opt) {
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
                  "templates/index-footer.html"), HTS_INDEX_FOOTER);
 
-  // initialiser mimedefs
-  //get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
-
   // Initialiser indexation
   if (opt->kindex)
     index_init(StringBuff(opt->path_html));
@@ -684,7 +720,6 @@ int httpmirror(char *url1, httrackp * opt) {
   opt->filters.filters = &filters;
   //
   opt->filters.filptr = &filptr;
-  //opt->filters.filter_max=&filter_max;
 
   // hash table
   opt->hash = &hash;
@@ -731,8 +766,7 @@ int httpmirror(char *url1, httrackp * opt) {
       else if (*a == '-')
         joker = 1;
 
-      if (joker) {              // joker ou filters
+      if (joker) { // joker ou filters
-        //char* p;
         char BIGSTK tempo[HTS_URLMAXSIZE * 2];
         int type;
         int plus = 0;
@@ -787,12 +821,11 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_extuninit;
               return 0;
             }
-            //opt->filters.filters=filters;
           }
 
         }
 
-      } else {                  // adresse normale
+      } else { // adresse normale
         char BIGSTK url[HTS_URLMAXSIZE * 2];
 
         // prochaine adresse
@@ -867,7 +900,6 @@ int httpmirror(char *url1, httrackp * opt) {
             htsbuff_cat(&primarybuff, "\n");
           }
         }
-        // fclose(fp);
         hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
                       StringBuff(opt->filelist));
 
@@ -922,7 +954,6 @@ int httpmirror(char *url1, httrackp * opt) {
 #endif
 
   // backing
-  //soc_max=opt->maxsoc;
   if (opt->maxsoc > 0) {
 #if BDEBUG==2
     _CLRSCR;
@@ -1236,15 +1267,10 @@ int httpmirror(char *url1, httrackp * opt) {
       // tester r.adr
       if (!error) {
         // erreur, pas de fichier chargé:
-        if ((!r.adr) && (r.is_write == 0)
+        if ((!r.adr) && (r.is_write == 0) && (r.statuscode != 301) &&
-            && (r.statuscode != 301)
+            (r.statuscode != 302) && (r.statuscode != 303) &&
-            && (r.statuscode != 302)
+            (r.statuscode != 307) && (r.statuscode != 412) &&
-            && (r.statuscode != 303)
+            (r.statuscode != 416)) {
-            && (r.statuscode != 307)
-            && (r.statuscode != 412)
-            && (r.statuscode != 416)
-          ) {
-          // error=1;
 
           // peut être que le fichier était trop gros?
           if ((istoobig
@@ -1326,30 +1352,8 @@ int httpmirror(char *url1, httrackp * opt) {
                                          ||may_be_hypertext_mime(opt, r.contenttype, urlfil()))   /* Is real media, .. */
           ) {
 
-          /* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
+          /* Convert charset to UTF-8 - NOT! (what about links ? remote server
-          //if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
+           * side will have troubles with converted names) */
-          //  char *charset;
-          //  char *pos;
-          //  if (r.charset[0] != '\0') {
-          //    charset = strdup(r.charset);
-          //  } else {
-          //    charset = hts_getCharsetFromMeta(r.adr, r.size);
-          //  }
-          //  if (charset != NULL) {
-          //    char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
-          //    /* Use new buffer */
-          //    if (utf8 != NULL) {
-          //      freet(r.adr);
-          //      r.size = strlen(utf8);
-          //      r.adr = utf8;
-          //      /* New UTF-8 charset */
-          //      r.charset[0] = '\0';
-          //      strcpy(r.charset, "utf-8");
-          //    }
-          //    /* Free charset */
-          //    free(charset);
-          //  }
-          //}
 
           /* Check bogus chars */
           if ((r.adr) && (r.size)) {
@@ -1490,50 +1494,6 @@ int httpmirror(char *url1, httrackp * opt) {
 
         }
       }
-      // MOVED IN back_finalize()
-      //
-      // -------------------- 
-      // REAL MEDIA HACK
-      // Check if we have to load locally the file
-      // --------------------
-      //if (!error) {
-      //  if (r.statuscode == HTTP_OK) {    // OK (ou 304 en backing)
-      //    if (r.adr==NULL) {    // Written file
-      //      if (may_be_hypertext_mime(r.contenttype, urlfil())) {   // to parse!
-      //        LLint sz;
-      //        sz=fsize_utf8(savename());
-      //        if (sz>0) {   // ok, exists!
-      //          if (sz < 8192) {   // ok, small file --> to parse!
-      //            FILE* fp=FOPEN(savename(),"rb");
-      //            if (fp) {
-      //              r.adr=malloct(sz + 1);
-      //              if (r.adr) {
-      //                if (fread(r.adr,1,sz,fp) == sz) {
-      //                  r.size=sz;
-      //                                                r.adr[sz] = '\0';
-      //                                                r.is_write = 0;
-      //                } else {
-      //                  freet(r.adr);
-      //                  r.size=0;
-      //                  r.adr = NULL;
-      //                  r.statuscode=STATUSCODE_INVALID;
-      //                  strcpybuff(r.msg, ".RAM read error");
-      //                }
-      //                fclose(fp);
-      //                fp=NULL;
-      //                // remove (temporary) file!
-      //                remove(savename());
-      //              }
-      //              if (fp)
-      //                fclose(fp);
-      //            }
-      //          }
-      //        }
-      //      }
-      //    }
-      //  }
-      //}
-      // EN OF REAL MEDIA HACK
 
       // ---stockage en cache---
       // stocker dans le cache?
@@ -1654,24 +1614,6 @@ int httpmirror(char *url1, httrackp * opt) {
       }
 #endif
 
-      /* info: updated */
-      /*
-         if (ptr>0) {
-         // "mis à jour"
-         if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) {    // page modifiée
-         if (strnotempty(savename())) {
-         HTS_STAT.stat_updated_files++;
-         //if ((opt->debug>0) && (opt->log!=NULL)) {
-         hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
-         }
-         } else {
-         if (!store_errpage) {
-         hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
-         }
-         }
-         }
-       */
-
       // ------------------------------------------------------
       // traitement (parsing)
       // ------------------------------------------------------
@@ -1717,13 +1659,8 @@ int httpmirror(char *url1, httrackp * opt) {
                 free(charset);
             }
             /* Could not detect charset: could it be UTF-8 ? */
-            /* No, we can not do that: browsers do not do it 
+            /* No, we can not do that: browsers do not do it
                (and it would break links). */
-            //if (page_charset[0] == '\0') {
-            //  if (is_unicode_utf8(r.adr, r.size)) {
-            //    strcpy(page_charset, "utf-8");
-            //  }
-            //}
             /* Could not detect charset */
             if (page_charset[0] == '\0') {
               hts_log_print(opt, LOG_INFO,
@@ -1801,10 +1738,6 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_uninit;
               return -1;
             }
-
-            // I'll have to segment this part
-// #include "htsparse.c"
-
           }
         }
         // Fin parsing HTML
@@ -1900,7 +1833,6 @@ int httpmirror(char *url1, httrackp * opt) {
              HTS_STAT.stat_files++;
              HTS_STAT.stat_bytes+=r.size;
            */
-          //printf("ok......\n");
         } else {
           // Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
           // récursion nous en empêche
@@ -1969,8 +1901,6 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          //printf("extern=%s\n",r.contenttype);
-
           // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
           if (r.adr != NULL || r.size == 0) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
@@ -2092,16 +2022,9 @@ int httpmirror(char *url1, httrackp * opt) {
           // prochain pass2
           while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
             ptr++;
-
-          //printf("first link==%d\n");
-
         }
       }
     }
-    // copy abort state if necessary from outside
-    //if (!exit_xh && opt->state.exit_xh) {
-    //  exit_xh=opt->state.exit_xh;
-    //}
     // a-t-on dépassé le quota?
     if (!back_checkmirror(opt)) {
       ptr = opt->lien_tot;
@@ -2137,47 +2060,7 @@ int httpmirror(char *url1, httrackp * opt) {
     hts_log_print(opt, LOG_NOTICE,
                   "No data seems to have been transferred during this session! : restoring previous one!");
     XH_uninit;
-    if ((fexist
+    hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/old.dat")))
-        &&
-        (fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-           "hts-cache/old.ndx")))) {
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.dat"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.ndx"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.lst"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.txt"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.dat"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.ndx"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.lst"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.txt"));
-    }
     opt->state.exit_xh = 2;     /* interrupted (no connection detected) */
     return 1;
   }
@@ -2298,13 +2181,11 @@ int httpmirror(char *url1, httrackp * opt) {
     char BIGSTK htstime[256];
     char BIGSTK infoupdated[256];
 
-    // int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
     LLint n =
       (LLint) (HTS_STAT.HTS_TOTAL_RECV /
                (max(1, time_local() - HTS_STAT.stat_timestart)));
 
     sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
-    //sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
     infoupdated[0] = '\0';
     if (opt->is_update) {
       if (HTS_STAT.stat_updated_files > 0) {
@@ -2387,12 +2268,6 @@ int httpmirror(char *url1, httrackp * opt) {
 
 */
 int engine_stats(void) {
-#if 0
-  static FILE *debug_fp = NULL; /* ok */
-
-  if (!debug_fp)
-    debug_fp = fopen("esstat.txt", "wb");
-#endif
   HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
   HTS_STAT.nb = 0;
   if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
@@ -2403,10 +2278,6 @@ int engine_stats(void) {
       if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
         TStamp dif;
 
-#if 0
-        fprintf(debug_fp, "set timer %d\n", i);
-        fflush(debug_fp);
-#endif
         dif = cdif - HTS_STAT.istat_timestart[i];
         if ((TStamp) (dif / 1000) > 0) {
           LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
@@ -2425,10 +2296,6 @@ int engine_stats(void) {
     // timer #0 resync timer #1 when reaching 1 second limit
     if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
       if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
-#if 0
-        fprintf(debug_fp, "resync timer 1\n");
-        fflush(debug_fp);
-#endif
         HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
         HTS_STAT.istat_timestart[1] = cdif;
         HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
@@ -2449,7 +2316,6 @@ void host_ban(httrackp * opt, int ptr,
   lien_back *const back = sback->lnk;
   const int back_max = sback->count;
 
-  //int l;
   int i;
 
   if (host[0] == '!')
@@ -2506,9 +2372,7 @@ void host_ban(httrackp * opt, int ptr,
   }
 
   // effacer liens
-  //l=strlen(host);
+  for (i = 0; i < opt->lien_tot; i++) {
-  for(i = 0; i < opt->lien_tot; i++) {
-    //if (heap(i)->adr_len==l) {    // même taille de chaîne
     // Calcul de taille sécurisée
     if (heap(i)) {
       if (heap(i)->adr) {
@@ -2547,7 +2411,6 @@ void host_ban(httrackp * opt, int ptr,
                     "WARNING! HostCancel detected memory leaks [null at %d]",
                     i);
     }
-    //}
   }
 }
 
@@ -2892,6 +2755,9 @@ int check_fatal_io_errno(void) {
 #endif
 #ifdef EROFS
   case EROFS:                  /* Read-only file system */
+#endif
+#ifdef EDQUOT
+  case EDQUOT: /* Disk quota exceeded */
 #endif
     return 1;
     break;
@@ -2961,7 +2827,6 @@ FILE *fileappend(filenote_strc * strc, const char *s) {
   // noter lst
   filenote(strc, s, NULL);
 
-  // if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s);    // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
   strcpybuff(fname, s);
 
 #if HTS_DOSNAME
@@ -3005,7 +2870,6 @@ int filecreateempty(filenote_strc * strc, const char *filename) {
 int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
   // gestion du fichier liste liste
   if (params) {
-    //filecreate_params* p = (filecreate_params*) params;
     strcpybuff(strc->path, params->path);
     strc->lst = params->lst;
     return 0;
@@ -3085,9 +2949,8 @@ void usercommand_exe(const char *cmd, const char *file) {
   }
 }
 
-static void postprocess_file(httrackp * opt, const char *save, const char *adr,
+static void postprocess_file(httrackp *opt, const char *save, const char *adr,
                              const char *fil) {
-  //int first = 0;
   /* MIME-html archive to build */
   if (opt != NULL && opt->mimehtml) {
     if (adr != NULL && strcmp(adr, "primary") == 0) {
@@ -3115,7 +2978,6 @@ static void postprocess_file(httrackp * opt, const char *save, const char *adr,
       }
 
       if (!opt->state.mimehtml_created) {
-        //first = 1;
         opt->state.mimefp =
           fopen(fconcat
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -3259,28 +3121,6 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
   return 0;
 }
 
-// vérifier taux de transfert
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate) {
-  // vérifier taux de transfert (pas trop grand?)
-  /*
-     if (maxrate>0) {
-     int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart));    // taux actuel de transfert
-     HTS_STAT.HTS_TOTAL_RECV_STATE=0;
-     if (r>maxrate) {    // taux>taux autorisé
-     int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
-     if (taux<15)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=1;   // ralentir un peu (<15% dépassement)
-     else if (taux<50)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=2;   // beaucoup (<50% dépassement)
-     else
-     HTS_STAT.HTS_TOTAL_RECV_STATE=3;   // énormément (>50% dépassement)
-     }
-     }
-   */
-}
-#endif
-
 // ---
 // sous routines liées au moteur et au backing
 
@@ -3290,21 +3130,8 @@ int backlinks_done(const struct_back * sback,
                    int ptr) {
   int n = 0;
 
-#if 0
-  int i;
-
-  //Links done and stored in cache
-  for(i = ptr + 1; i < lien_tot; i++) {
-    if (heap(i)) {
-      if (heap(i)->pass2 == -1) {
-        n++;
-      }
-    }
-  }
-#else
   // finalized in background
   n += HTS_STAT.stat_background;
-#endif
   n += back_done_incache(sback);
   return n;
 }
@@ -3441,8 +3268,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
     /* on a déja parcouru */
     if (p < cache->ptr_ant)
       p = cache->ptr_ant;
-    while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
+    while (p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
-      //while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
       int ok = 1;
 
       // on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
@@ -3454,12 +3280,6 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
           ok = 0;
       }
 
-      // Why in hell did I do that ?
-      //if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0' 
-      //  && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0)     // lookup in liens_record
-      //{
-      //  ok = 0;
-      //}
       if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
           || hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
         ok = 0;
@@ -3488,7 +3308,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
         }
       }
       p++;
-    }                           // while
+    } // while
     /* sauver position dernière anticipation */
     cache->ptr_ant = p;
     cache->ptr_last = ptr;
@@ -3750,20 +3570,7 @@ HTSEXT_API hts_boolean hts_has_stopped(httrackp *opt) {
   return ended;
 }
 
-// régler en cours de route les paramètres réglables..
-// -1 : erreur
-//HTSEXT_API int hts_setopt(httrackp* set_opt) {
-//  if (set_opt) {
-//    httrackp* engine_opt=hts_declareoptbuffer(NULL);
-//    if (engine_opt) {
-//      //_hts_setopt=opt;
-//      copy_htsopt(set_opt,engine_opt);
-//    }
-//  }
-//  return 0;
-//}
 // ajout d'URL
-// -1 : erreur
 HTSEXT_API hts_boolean hts_addurl(httrackp *opt, char **url) {
   if (url)
     opt->state._hts_addurl = url;
@@ -4024,10 +3831,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->link_import = 0; // pas mode import
 
               // écrire autres paramètres de la structure-lien
-              //if (meme_adresse)                                 
               heap_top()->premier = heap(ptr)->premier;
-              //else    // sinon l'objet père est le précédent lui même
-              //  heap_top()->premier=ptr;
 
               heap_top()->precedent = ptr;
               // noter la priorité
@@ -4038,9 +3842,6 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->pass2 = max(pass_fix, numero_passe);
               heap_top()->retry = opt->retry;
 
-              //strcpybuff(heap_top()->adr,adr);
-              //strcpybuff(heap_top()->fil,fil);
-              //strcpybuff(heap_top()->sav,save); 
               hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
                             heap_top()->adr, heap_top()->fil,
                             heap_top()->sav);

src/htscore.h

@@ -216,6 +216,7 @@ struct cache_back {
   int zipEntriesCapa;
   hts_boolean
       zipWriteFailed; /**< a cache write failed; stop touching the stream */
+  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
 };
 
 #ifndef HTS_DEF_FWSTRUCT_hash_struct
@@ -411,10 +412,6 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate);
-#endif
-
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
@@ -466,11 +463,6 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
-// Java
-#if 0
-int hts_add_file(char *file, int file_position);
-#endif
-
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);

src/htscoremain.c

@@ -145,7 +145,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   char *url = NULL;             // URLS séparées par un espace
   int url_sz = 65535;
 
-  //char url[65536];         // URLS séparées par un espace
   // the parametres
   int httrack_logmode = 3;      // ONE log file
 
@@ -290,12 +289,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         if (tmp_argc == 1) {    /* pas -P & co */
           if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
             if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-            //if (argv_url>=0) {
+                            // previous options
             argv_url++;
             if (!argv_firsturl)
               argv_firsturl = x_argv[x_argc - 1];
-            //}
           } else {
             if (strcmp(tmp_argv[0], "-h") == 0) {
               help(argv[0], !opt->quiet);
@@ -324,8 +322,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         } else if (tmp_argc == 2) {
           if ((strcmp(tmp_argv[0], "-%L") == 0)) {      // liste d'URLs
             if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-            //if (argv_url>=0)
+                            // previous options
             argv_url++;         /* forcer */
           }
         }
@@ -403,15 +401,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     else
                       inQuote = 1;
                   } else if (!inQuote && !noDbl && argv[na][i] == ',') {
-                    //StringAddchar(path, '\0');
-                    //j = 0;
                     path = &opt->path_log;
                   } else {
                     StringAddchar(*path, argv[na][i]);
-                    //path[j++] = argv[na][i];
                   }
                 }
-                //path[j++] = '\0';
                 if (StringLength(opt->path_log) == 0) {
                   StringCopyS(opt->path_log, opt->path_html);
                 }
@@ -420,7 +414,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if (check_path(&opt->path_html, argv_firsturl)) {
                   opt->dir_topindex = 1;        // rebuilt top index
                 }
-                //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));                
               }
               break;
             }                   // switch
@@ -539,74 +532,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             insert_after++;
           }
         }
-      } while(lastp != NULL);
+      } while (lastp != NULL);
-      //fclose(fp);
     }
   }
 
-  // Existence d'un cache - pas de new mais un old.. renommer
+  // No new cache but an old one? promote it
 #if DEBUG_STEPS
   printf("Checking cache\n");
 #endif
-  if (!fexist
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-      (fconcat
-       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-       StringBuff(opt->path_log), "hts-cache/new.zip"))) {
-    if (fexist
-        (fconcat
-         (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-         StringBuff(opt->path_log), "hts-cache/old.zip"))) {
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.zip"));
-    }
-  } else
-    if ((!fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log), "hts-cache/new.dat")))
-        ||
-        (!fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log),
-           "hts-cache/new.ndx")))) {
-    if ((fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log), "hts-cache/old.dat")))
-        &&
-        (fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log),
-           "hts-cache/old.ndx")))) {
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/new.dat"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/new.ndx"));
-      //remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.dat"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.ndx"));
-      //rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
-    }
-  }
 
   /* Interrupted mirror detected */
   if (!opt->quiet) {
@@ -774,24 +708,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             HTS_PANIC_PRINTF(s);
             htsmain_free();
             return -1;
-#else
-#if 0
-            char _args[8][256];
-            char *args[8];
-
-            printf("Cheking for updates...\n");
-            strcpybuff(_args[0], argv[0]);
-            strcpybuff(_args[1], "--get");
-            sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
-            strcpybuff(_args[3], "--quickinfo");
-            args[0] = _args[0];
-            args[1] = _args[1];
-            args[2] = _args[2];
-            args[3] = _args[3];
-            args[4] = NULL;
-            if (execvp(args[0], args) == -1) {
-            }
-#endif
 #endif
           }
           //
@@ -809,103 +725,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     }
   }
 
-  // Compter urls/jokers
-  /*
-     if (argv_url<=0) { 
-     int na;
-     argv_url=0;
-     for(na=1;na<argc;na++) {
-     if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
-     na++;    // sauter nom de proxy
-     } else if (!cmdl_opt(argv[na])) { 
-     argv_url++;   // un de plus       
-     } else if (strcmp(argv[na],"-h")==0) {
-     help(argv[0],!opt->quiet);
-     htsmain_free();
-     return 0;
-     } else {
-     if ((strchr(argv[na],'q')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     if ((strchr(argv[na],'i')!=NULL)) {  // doit.log!
-     argv_url=0;
-     na=argc;
-     }
-     }
-     }
-     }  
-   */
-
-  // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
-  // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
-  /*
-     if (argv_url==0) {
-     //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) {  // il existe déja un cache précédent.. renommer
-     if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) {    // un cache est présent
-
-     x_argvblk=(char*) calloct(32768,1);
-
-     if (x_argvblk!=NULL) {
-     FILE* fp;
-     int x_argc;
-
-     //strcpybuff(x_argvblk,"httrack ");
-     fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
-     if (fp) {
-     linput(fp,x_argvblk+strlen(x_argvblk),8192);
-     fclose(fp); fp=NULL;
-     }
-
-     // calculer arguments selon derniers arguments
-     x_argv[0]=argv[0];
-     x_argc=1;
-     {
-     char* p=x_argvblk;
-     do {
-     x_argv[x_argc++]=p;
-     //p=strstr(p," ");
-     // exemple de chaine: "echo \"test\"" c:\a "\$0"
-     p=next_token(p,1);    // prochain token
-     if (p) {
-     *p=0;    // octet nul (tableau)
-     p++;
-     }            
-     } while(p!=NULL);
-     }
-     // recopier arguments actuels (pointeurs uniquement)
-     {
-     int na;
-     for(na=1;na<argc;na++) {
-     if (strcmp(argv[na],"-O") != 0)    // SAUF le path!
-     x_argv[x_argc++]=argv[na];
-     else
-     na++;
-     }
-     }
-     argc=x_argc;      // nouvel argc
-     argv=x_argv;      // nouvel argv
-     }
-
-     }
-     //}
-     }
-   */
-
-  // Vérifier quiet
-  /*
-     { 
-     int na;    
-     for(na=1;na<argc;na++) {
-     if (!cmdl_opt(argv[na])) { 
-     if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
-     na++;    // sauter nom de proxy
-     } else {
-     if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     }
-     }
-     }
-   */
-
   /* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
      here, ahead of the no-URL usage gate below, so they need no dummy URL. */
   {
@@ -1012,8 +831,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   } else {                      // plus de 2 paramètres
     // un fichier log existe?
     if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-      StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // fichier lock?
+                       StringBuff(opt->path_log),
-      //char s[32];
+                       "hts-in_progress.lock"))) { // fichier lock?
 
       opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
       if (opt->quiet == 0) {
@@ -1044,11 +863,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           }
         }
       }
-    } else
+    } else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-      if (fexist
+                              StringBuff(opt->path_html), "index.html"))) {
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
-      //char s[32];
       opt->cache = HTS_CACHE_TEST_UPDATE;
       if (opt->quiet == 0) {
         if ((fexist
@@ -1143,7 +959,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             //
           case 'g':            // récupérer un (ou plusieurs) fichiers isolés
             opt->wizard = HTS_WIZARD_AUTO;
-            //opt->wizard=0;             // pas de wizard
             opt->cache = HTS_CACHE_NONE; // ni de cache
             opt->makeindex = 0; // ni d'index
             httrack_logmode = 1;        // erreurs à l'écran
@@ -1243,9 +1058,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             while(isdigit((unsigned char) *(com + 1)))
               com++;
             break;
-            //
-            //case 'A': opt->urlmode=1; break;
-            //case 'R': opt->urlmode=2; break;
           case 'K':
             opt->urlmode = HTS_URLMODE_ABSOLUTE;
             if (isdigit((unsigned char) *(com + 1))) {
@@ -1257,8 +1069,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               while(isdigit((unsigned char) *(com + 1)))
                 com++;
             }
-            //if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
-            //
           case 'c':
             if (isdigit((unsigned char) *(com + 1))) {
               sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1746,7 +1556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   } else {
                     char *a;
 
-                    //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
                     for(a = argv[na]; *a != '\0'; a++) {
                       if (*a == ';') {  /* next one */
                         StringAddchar(opt->mimedefs, '\n');
@@ -2085,10 +1894,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   HTS_PANIC_PRINTF(s);
                   htsmain_free();
                   return -1;
-                }
+              } break;
-                break;
-
-                //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
               }
             }
             break;
@@ -2332,15 +2138,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 return 0;
                 break;
               case '~':        /* internal lib test */
-                HTS_PANIC_PRINTF
+                HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
-                  ("Option #~ is disabled for security reasons");
-                //Disabled because choke on GCC 4.3 (toni from links2linux.de)
-                //{
-                //  char thisIsATestYouShouldSeeAnError[12];
-                //  const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
-                //  strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
-                //  return 0;
-                //}
                 break;
               case 'f':
                 opt->flush = 1;
@@ -2442,7 +2240,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               opt->proxy.active = 1;
               // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
               a = argv[na] + strlen(argv[na]) - 1;
-              // a=strstr(argv[na],":");  // port
               while((a > argv[na]) && (*a != ':') && (*a != '@'))
                 a--;
               if (*a == ':') {  // un port est présent, <proxy>:port
@@ -2547,116 +2344,14 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 #endif
 #endif
 
-  //printf("WARNING! This is *only* a beta-release of HTTrack\n");
   io_flush;
 
 #if DEBUG_STEPS
   printf("Cache & log settings\n");
 #endif
 
-  // on utilise le cache..
+  // If both cache generations exist, keep the most complete one
-  // en cas de présence des deux versions, garder la version la plus avancée,
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  // cad la version contenant le plus de fichiers  
-  if (opt->cache) {
-    if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // problemes..
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-            "hts-cache/new.dat"))) {
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.zip"))) {
-          if (fsize
-              (fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                "hts-cache/new.zip")) < 32768) {
-            if (fsize
-                (fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.zip")) > 65536) {
-              if (fsize
-                  (fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                          StringBuff(opt->
-                                                                     path_log),
-                                                          "hts-cache/new.zip")))
-              {
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.zip"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.zip"));
-              }
-            }
-          }
-        }
-      } else
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.dat"))
-            &&
-            fexist(fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/new.ndx"))) {
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat"))
-            &&
-            fexist(fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.ndx"))) {
-          // switcher si new<32Ko et old>65Ko (tailles arbitraires) ?
-          // ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre
-          // alors l'ancien cache
-          if (fsize
-              (fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                "hts-cache/new.dat")) < 32768) {
-            if (fsize
-                (fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.dat")) > 65536) {
-              if (fsize
-                  (fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                          StringBuff(opt->
-                                                                     path_log),
-                                                          "hts-cache/new.dat")))
-              {
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.dat"));
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.ndx"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.dat"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.ndx"));
-                //} else {  // ne rien faire
-                //  remove("hts-cache/old.dat");
-                //  remove("hts-cache/old.ndx");
-              }
-            }
-          }
-        }
-      }
-    }
-  }
   // Débuggage des en têtes
   if (_DEBUG_HEAD) {
     ioinfo =
@@ -2732,7 +2427,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     {
       FILE *fp = NULL;
 
-      //int n=0;
       char t[256];
 
       time_local_rfc822(t);     // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2766,9 +2460,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       }
 
       strcpybuff(n_lock,
-             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                     "hts-in_progress.lock"));
+                         StringBuff(opt->path_log), "hts-in_progress.lock"));
-      //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
       /*do {
          if (!n)
          sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2844,8 +2537,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           fprintf(fp, LF);
           fclose(fp);
           fp = NULL;
-          //} else if (opt->debug>1) {
-          //  printf("! FileOpen error, \"%s\"\n",strerror(errno));
         }
       }
       // petit message dans le lock

src/htsfilters.c

@@ -102,9 +102,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
 // cet algo est 'un peu' récursif mais ne consomme pas trop de tm
 // * = toute lettre
 // --?-- : spécifique à HTTrack et aux ?
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                          int *size_flag) {
+                                LLint *size, int *size_flag) {
-  //int err=0;
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -236,9 +235,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
-
               }
-              // else err=1;
               i += 3;
             } else { // 1 car, ex: *[ ]
               pass[(int) (unsigned char) joker[i]] = 1;
@@ -262,7 +259,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
-      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {

src/htsftp.c

@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
-// #if HTS_INET6==0
 #define FTP_PASV 1
-// #else
-// no passive mode for v6
-// #define FTP_PASV 0
-// #endif
 
 #define FTP_DEBUG 0
-//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -206,20 +200,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   {
     char *a;
 
-#if 0
-    a = back->url_fil + strlen(back->url_fil) - 1;
-    while((a > back->url_fil) && (*a != '/'))
-      a--;
-    if (*a != '/') {
-      a = NULL;
-    }
-#else
     a = back->url_fil;
-#endif
     if (a != NULL && *a != '\0') {
-#if 0
-      a++;                      // sauter /
-#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -243,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -277,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -290,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
-    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -345,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
-#if 0
-            // --CWD--
-            char *a;
-
-            a = back->url_fil + strlen(back->url_fil) - 1;
-            while((a > back->url_fil) && (*a != '/'))
-              a--;
-            if (*a == '/') {    // ok repéré
-              char BIGSTK target[1024];
-
-              target[0] = '\0';
-              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
-              if (strnotempty(target) == 0)
-                strcatbuff(target, "/");
-              strcpybuff(back->info, "cwd");
-              snprintf(line, sizeof(line), "CWD %s", target);
-              send_line(soc_ctl, line);
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-              _CHECK_HALT_FTP;
-              if (line[0] == '2') {
-                send_line(soc_ctl, "TYPE I");
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (line[0] == '2') {
-                  // ok..
-                } else {
-                  strcpybuff(back->r.msg, "TYPE I error");
-                  // back->status=STATUS_FTP_READY;    // fini
-                  back->r.statuscode = STATUSCODE_INVALID;
-                }
-              } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
-                back->r.statuscode = STATUSCODE_INVALID;
-              }                 // sinon on est prêts
-            } else {
-              strcpybuff(back->r.msg, "Unexpected ftp error");
-              // back->status=STATUS_FTP_READY;    // fini
-              back->r.statuscode = STATUSCODE_INVALID;
-            }
-#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
-          // back->status=STATUS_FTP_READY;    // fini
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
-        // back->status=STATUS_FTP_READY;    // fini
+                 linejmp(line));
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -469,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -501,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
-              // back->status=STATUS_FTP_READY;    // fini
+                       linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -613,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
+                  snprintf(back->r.msg, sizeof(back->r.msg),
-                          linejmp(line));
+                           "RETR command error: %s", linejmp(line));
-                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -626,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-          // back->status=STATUS_FTP_READY;    // fini
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -663,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg),
-              // back->status=STATUS_FTP_READY;    // fini
+                       "RETR command error: %s", linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -683,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
-          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -707,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
-            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
-
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
-                // back->status=STATUS_FTP_READY;    // fini
+                         timeout);
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -743,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
-                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
-                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
-                } else {        // Erreur ou terminé
+                } else { // Erreur ou terminé
-                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -762,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
-                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -771,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -787,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
-                // back->status=STATUS_FTP_READY;    // fini
+                         linejmp(line));
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -822,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
-  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -989,9 +907,7 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
-    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
-      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
       if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
@@ -1118,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
-    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-10"
+#define HTTRACK_VERSION "3.49-11"
-#define HTTRACK_VERSIONID "3.49.10"
+#define HTTRACK_VERSIONID "3.49.11"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 

src/htshelp.c

@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
-  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
-  //char *mainpath = (char *) malloct(256);
-  //char *projname = (char *) malloct(256);
-  //char *stropt = (char *) malloct(2048);        // options
-  //char *stropt2 = (char *) malloct(2048);       // options longues
-  //char *strwild = (char *) malloct(2048);       // wildcards
-  //char *cmd = (char *) malloct(4096);
-  //char *str = (char *) malloct(256);
-  //char **argv = (char **) malloct(256 * sizeof(char *));
-
   //
   char *a;
 
@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
-  // Créer si ce n'est fait un index.html 1er niveau
-  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
-    //} else {
-    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
-// Créer un index.html vide
-void make_empty_index(const char *str) {
-#if 0
-  if (!fexist(fconcat(str, "index.html"))) {
-    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
-
-    if (fp) {
-      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
-      fprintf(fp,
-              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
-              CRLF);
-      fclose(fp);
-    }
-  }
-#endif
-}
-
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -808,7 +777,4 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
-
-//  infomsg("  R  *relative links (e.g ../link)\n");
-//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
-void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
-  //
-  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
-    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
-    //  inscript=0;
   } else
     return 0;
 
@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
-    //FILE* fp=NULL;
-    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
-            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
-            //}
           }
         }
       }
-      //fclose(fp);
     }
   }
 
@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
-    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
-                        //else                // HTML
-                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
-                          //else                // HTML
-                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -175,15 +175,14 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
 #if JAVADEBUG
       printf("fopen\n");
 #endif
-      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
-        //fprintf(stderr, "Cannot open input file.\n");
+          NULL) {
         sprintf(str->err_msg, "Unable to open file %s", file);
         return 0;               // une erreur..
       }
 #if JAVADEBUG
       printf("fread\n");
 #endif
-      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
       if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
         fclose(fpout);
         sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
@@ -257,12 +256,10 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
       printf("addfiles\n");
 #endif
       {
-        //unsigned int acess;
         unsigned int Class;
         unsigned int SClass;
         int i;
 
-        //acess = readshort(fpout);
         Class = readshort(fpout);
         SClass = readshort(fpout);
 
@@ -407,11 +404,9 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
 
       p = &buffer[0];
 
-      //fflush(fp); 
       trans.file_position = ftell(fp);
       length = readshort(fp);
       if (length < HTS_URLMAXSIZE) {
-        // while ((length > 0) && (length<500)) {
         while(length > 0) {
           *p++ = fgetc(fp);
 
@@ -419,17 +414,13 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
         }
         *p = '\0';
 
-        //#if JDEBUG
-        //      if(tris(buffer)==1) printf("%s\n ",buffer);
-        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
-        //#endif
         if (tris(str->opt, buffer) == 1)
           str->addLink(str, buffer);    /* trans.file_position */
         else if (tris(str->opt, buffer) == 2)
           str->addLink(str, printname(rname, buffer));
 
         strcpy(trans.name, buffer);
-      } else {                  // gros pb
+      } else { // gros pb
         while((length > 0) && (!feof(fp))) {
           fgetc(fp);
           length--;
@@ -445,7 +436,6 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
     }
     break;
   default:
-    // printf("Type inconnue\n");
     // on arrête tout 
     sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
     *error = 1;
@@ -508,7 +498,6 @@ static char *printname(char rname[1024], char name[1024]) {
     return rname;  // ""
   }
   p += 2;
-  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
   p1 = rname;
   for(j = 0; name[j] != '\0'; j++, p++) {
     if (*p == '/')

src/htslib.c

@@ -598,56 +598,27 @@ static const char *hts_mime_modern[][2] = {
 
 // Reserved (RFC2396)
 #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
-#define CHAR_RESERVED(c)  ( CIS(c,';') \
+#define CHAR_RESERVED(c)                                                       \
-                         || CIS(c,'/') \
+  (CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') ||  \
-                         || CIS(c,'?') \
+   CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
-                         || CIS(c,':') \
-                         || CIS(c,'@') \
-                         || CIS(c,'&') \
-                         || CIS(c,'=') \
-                         || CIS(c,'+') \
-                         || CIS(c,'$') \
-                         || CIS(c,',') )
-//#define CHAR_RESERVED(c)  ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
 // Delimiters (RFC2396)
-#define CHAR_DELIM(c)     ( CIS(c,'<') \
+#define CHAR_DELIM(c)                                                          \
-                         || CIS(c,'>') \
+  (CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
-                         || CIS(c,'#') \
-                         || CIS(c,'%') \
-                         || CIS(c,'\"') )
-//#define CHAR_DELIM(c)     ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
 // Unwise (RFC2396)
-#define CHAR_UNWISE(c)    ( CIS(c,'{') \
+#define CHAR_UNWISE(c)                                                         \
-                         || CIS(c,'}') \
+  (CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
-                         || CIS(c,'|') \
+   CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
-                         || CIS(c,'\\') \
-                         || CIS(c,'^') \
-                         || CIS(c,'[') \
-                         || CIS(c,']') \
-                         || CIS(c,'`') )
-//#define CHAR_UNWISE(c)    ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
 // Special (escape chars) (RFC2396 + >127 )
 #define CHAR_LOW(c)       ( ((unsigned char)(c) <= 31) )
 #define CHAR_HIG(c)       ( ((unsigned char)(c) >= 127) )
 #define CHAR_SPECIAL(c)   ( CHAR_LOW(c) || CHAR_HIG(c) )
 // We try to avoid them and encode them instead
-#define CHAR_XXAVOID(c)   ( CIS(c,' ') \
+#define CHAR_XXAVOID(c)                                                        \
-                         || CIS(c,'*') \
+  (CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') ||               \
-                         || CIS(c,'\'') \
+   CIS(c, '&') || CIS(c, '!'))
-                         || CIS(c,'\"') \
+#define CHAR_MARK(c)                                                           \
-                         || CIS(c,'&') \
+  (CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') ||  \
-                         || CIS(c,'!') )
+   CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))
-//#define CHAR_XXAVOID(c)   ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
-#define CHAR_MARK(c)      ( CIS(c,'-') \
-                         || CIS(c,'_') \
-                         || CIS(c,'.') \
-                         || CIS(c,'!') \
-                         || CIS(c,'~') \
-                         || CIS(c,'*') \
-                         || CIS(c,'\'') \
-                         || CIS(c,'(') \
-                         || CIS(c,')') )
-//#define CHAR_MARK(c)      ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
 
 // conversion éventuelle / vers antislash
 #ifdef _WIN32
@@ -841,15 +812,12 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
 // treat: traiter header?
 // waitconnect: attendre le connect()
 // note: dans retour, on met les params du proxy
-T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
+T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
-                  const char *xsend, const char *adr, const char *fil, htsblk * retour) {
+                  const char *xsend, const char *adr, const char *fil,
-  //htsblk retour;
+                  htsblk *retour) {
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   T_SOC soc = INVALID_SOCKET;
   char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];
 
-  //char *p,*q;
-
   // retour prédéfini: erreur
   if (retour) {
     retour->adr = NULL;
@@ -934,7 +902,6 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
         strcpybuff(retour->msg, "Unable to open local file");
       else {
         // Note: On passe par un FILE* (plus propre)
-        //soc=open(fil,O_RDONLY,0);    // en lecture seule!
         retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
           unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb");      // ouvrir
         if (retour->fp == NULL)
@@ -1011,17 +978,10 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
 
         } while(strnotempty(rcvd));
 
-        //rcvsize=-1;    // forCER CHARGEMENT INCONNU
+      } else { // si GET, on recevra l'en tête APRES
-
-        //if (retour)
-        //  retour->totalsize=rcvsize;
-
-      } else {                  // si GET, on recevra l'en tête APRES
-        //rcvsize=-1;    // on ne connait pas la taille de l'en-tête
         if (retour)
           retour->totalsize = -1;
       }
-
     }
 
   }
@@ -1118,19 +1078,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
   char BIGSTK buffer_head_request[16384];
   buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };
 
-  //int use_11=0;     // HTTP 1.1 utilisé
   int direct_url = 0;           // ne pas analyser l'url (exemple: ftp://)
   const char *search_tag = NULL;
 
   // Initialize buffer
   buffer_head_request[0] = '\0';
 
-  // header Date
-  //strcatbuff(buff,"Date: ");
-  //time_gmt_rfc822(buff);    // obtenir l'heure au format rfc822
-  //sendc("\n");
-  //strcatbuff(buff,buff);
-
   // possibilité non documentée: >post: et >postfile:
   // si présence d'un tag >post: alors executer un POST
   // exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
@@ -1224,11 +1177,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
     }
 
     // protocole
-    if (!retour->req.http11) {  // forcer HTTP/1.0
+    if (!retour->req.http11) { // forcer HTTP/1.0
-      //use_11=0;
       print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
-    } else {                    // Requète 1.1
+    } else { // Requète 1.1
-      //use_11=1;
       print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
     }
 
@@ -1364,9 +1315,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
         }
       }
-
     }
-    //strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");
 
     // Custom header(s)
     if (strnotempty(retour->req.headers)) {
@@ -1520,14 +1469,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a += strlen("filename=");
         while(is_space(*a))
           a++;
-        //a=strchr(a,'"');
         if (a) {
           char *c = NULL;
 
-          //a++;      /* jump " */
           while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
             a = c + 1;
-          //b=strchr(a+1,'"');
           b = a + strlen(a) - 1;
           while(is_space(*b))
             b--;
@@ -1544,16 +1490,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
   } else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
-    if ((int) strlen(rcvd + p) < 64) {  // pas trop long?
+    if ((int) strlen(rcvd + p) < 64) { // pas trop long?
-      //struct tm* tm_time=convert_time_rfc822(rcvd+p);
       strcpybuff(retour->lastmodified, rcvd + p);
     }
   } else if ((p = strfield(rcvd, "Date:")) != 0) {
     if (strnotempty(retour->lastmodified) == 0) {       /* pas encore de last-modified */
       while(is_realspace(*(rcvd + p)))
         p++;                    // sauter espaces
-      if ((int) strlen(rcvd + p) < 64) {        // pas trop long?
+      if ((int) strlen(rcvd + p) < 64) { // pas trop long?
-        //struct tm* tm_time=convert_time_rfc822(rcvd+p);
         strcpybuff(retour->lastmodified, rcvd + p);
       }
     }
@@ -1566,14 +1510,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       else                      // erreur.. ignorer
         retour->etag[0] = '\0';
     }
-  }
+  } else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
-  // else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) {  // chunk!
-  else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) {   // chunk!
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
     if (strfield(rcvd + p, "chunked")) {
-      retour->is_chunk = 1;     // chunked
+      retour->is_chunk = 1; // chunked
-      //retour->http11=2;     // chunked
 #if HDEBUG
       printf("ok, Transfer-Encoding: détecté\n");
 #endif
@@ -1750,7 +1691,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           retour->location[0] = '\0';
       }
     }
-  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) {  // ohh un cookie
+  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
+             (cookie)) {        // ohh un cookie
     char *a = rcvd + p;         // pointeur
     char domain[256];           // domaine cookie (.netscape.com)
     char path[256];             // chemin (/)
@@ -1794,10 +1736,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
             a++;                // sauter espaces
           value_st = a;
           while((*a != ';') && (*a))
-            a++;                // prochain ;
+            a++; // prochain ;
-          //while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++;    // prochain " (et pas \")
           value_end = a;
-          //if (*a==';') {  // finit par un ;
           // vérifier débordements
           if ((((int) (token_end - token_st)) < 200)
               && (((int) (value_end - value_st)) < 8000)
@@ -2139,8 +2079,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
             nl = READ_ERROR;
           }
         }
-        //if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
-        //  nl=-1;  // break
 
         // libérer bloc tempo
         freet(buff);
@@ -2228,9 +2166,6 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
   T_SOC soc;
   htsblk retour;
 
-  //int rcvsize=-1;
-  //char* rcv=NULL;    // adresse de retour
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   TStamp tl;
   int timeout = 30;             // timeout pour un check (arbitraire) // **
 
@@ -2239,11 +2174,8 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
 
   loc[0] = '\0';
   hts_init_htsblk(&retour);
-  //memset(&retour, 0, sizeof(htsblk));    // effacer
   retour.location = loc;        // si non nul, contiendra l'adresse véritable en cas de moved xx
 
-  //soc=http_fopen(adr,fil,&retour,NULL);  // ouvrir, + header
-
   // on ouvre en head, et on traite l'en tête
   soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour);     // ouvrir HEAD, + envoi header
 
@@ -2619,7 +2551,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
 
     // chemin www... trop long!!
     if ((((int) (q - p))) > HTS_URLMAXSIZE) {
-      //strcpybuff(retour.msg,"Path too long");
       return -1;                // erreur
     }
     // recopier adrfil->adresse www..
@@ -2805,7 +2736,6 @@ void deletesoc_r(htsblk * r) {
 #if HTS_USEOPENSSL
   if (r->ssl_con) {
     SSL_shutdown(r->ssl_con);
-    // SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
     SSL_free(r->ssl_con);
     r->ssl_con = NULL;
   }
@@ -3194,7 +3124,6 @@ int finput(T_SOC fd, char *s, int max) {
   int j = 0;
 
   do {
-    //c=fgetc(fp);
     if (read((int) fd, &c, 1) <= 0) {
       c = 0;
     }
@@ -3210,7 +3139,7 @@ int finput(T_SOC fd, char *s, int max) {
         break;
       }
     }
-  } while((c != 0) && (j < max - 1));
+  } while ((c != 0) && (j < max - 1));
   s[j] = '\0';
   return j;
 }
@@ -4869,7 +4798,6 @@ LLint check_downloadable_bytes(int rate) {
 
     time_now = mtime_local();
     elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
-    // NO totally stupid - elapsed_useconds+=1000;      // for the next second, too
     bytes_transferred_during_period =
       (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);
 
@@ -4882,32 +4810,6 @@ LLint check_downloadable_bytes(int rate) {
     return TAILLE_BUFFER;
 }
 
-//
-// 0 : OK
-// 1 : slow down
-#if 0
-int HTS_TOTAL_RECV_CHECK(int var) {
-  if (HTS_STAT.HTS_TOTAL_RECV_STATE)
-    return 1;
-  /*
-     {
-     if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { 
-     var = min(var,32); 
-     Sleep(250); 
-     } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { 
-     var = min(var,256); 
-     Sleep(100); 
-     } else { 
-     var/=2; 
-     if (var<=0) var=1; 
-     Sleep(50); 
-     } 
-     }
-   */
-  return 0;
-}
-#endif
-
 // Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
 // returns: 
 // >0 : data received
@@ -4916,7 +4818,6 @@ int HTS_TOTAL_RECV_CHECK(int var) {
 int hts_read(htsblk * r, char *buff, int size) {
   int retour;
 
-  //  return read(soc,buff,size);
   if (r->is_file) {
 #if HTS_WIDE_DEBUG
     DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
@@ -4934,7 +4835,6 @@ int hts_read(htsblk * r, char *buff, int size) {
     if (r->soc == INVALID_SOCKET)
       printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
 #endif
-    //HTS_TOTAL_RECV_CHECK(size);         // Diminuer au besoin si trop de données reçues
 #if HTS_USEOPENSSL
     if (r->ssl) {
       retour = SSL_read(r->ssl_con, buff, size);
@@ -5369,18 +5269,13 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)
 
 // --- Tracage des mallocs() ---
 #ifdef HTS_TRACE_MALLOC
-//#define htsLocker(A, N) htsLocker(A, N)
 #define htsLocker(A, N) do {} while(0)
 static mlink trmalloc = { NULL, 0, 0, NULL };
 
 static int trmalloc_id = 0;
 static htsmutex *mallocMutex = NULL;
-static void hts_meminit(void) {
+
-  //if (mallocMutex == NULL) {
+static void hts_meminit(void) {}
-  //  mallocMutex = calloc(sizeof(*mallocMutex), 1);
-  //  htsLocker(mallocMutex, -999);
-  //}
-}
 void *hts_malloc(size_t len) {
   void *adr;
 
@@ -5464,7 +5359,6 @@ void hts_free(void *adr) {
               htsboundary);
       lnk->next = lnk->next->next;
       free((void *) blk_free);
-      //blk_free->id=-1;
       free((char *) adr - bsize);
       htsLocker(mallocMutex, 0);
       return;
@@ -5533,7 +5427,6 @@ mlink *hts_find(char *adr) {
 
     if (depl < 0)
       depl = -depl;
-    //assertf(depl < 512000);   /* near the stack frame.. doesn't look like malloc but stack variable */
     return NULL;
   }
 }
@@ -5589,8 +5482,7 @@ int ftp_available(void) {
 }
 #else
 int ftp_available(void) {
-  return 1;                     // ok!
+  return 1; // ok!
-  //return 0;   // SOUS UNIX, PROBLEMESs
 }
 #endif
 
@@ -5804,7 +5696,6 @@ HTSEXT_API int hts_init(void) {
       assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
     }
 
-    // OpenSSL_add_all_algorithms();
     openssl_ctx = SSL_CTX_new(method);
     if (!openssl_ctx) {
       fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
@@ -6126,7 +6017,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->log = stdout;
   opt->errlog = stderr;
   opt->flush = HTS_TRUE;
-  // opt->aff_progress=0;
   opt->keyboard = HTS_FALSE;
   //
   StringCopy(opt->path_html, "");
@@ -6136,8 +6026,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   //
   opt->maxlink = 100000;        // 100,000 liens max par défaut
   opt->maxfilter = 200;         // 200 filtres max par défaut
-  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
+  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
-  //opt->maxcache_anticipate=256;  // maximum de liens à anticiper
+                                // PARAMETRABLE POUR L'INSTANT --
   opt->maxtime = -1;            // temps max en secondes
   opt->maxrate = 100000;        // taux maxi
   opt->maxconn = 5.0;           // nombre connexions/s
@@ -6290,11 +6180,9 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
 }
 
 // defaut wrappers
-static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
+static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
-}
+
-static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
+static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
-  // hts_freevar();
-}
 static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
   return 1;
 }

src/htslib.h

@@ -171,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
 // fonctions unix/winsock
 int hts_read(htsblk * r, char *buff, int size);
 
-//int HTS_TOTAL_RECV_CHECK(int var);
 LLint check_downloadable_bytes(int rate);
 
 HTSEXT_API int hts_uninit_module(void);
@@ -190,7 +189,6 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
 int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
                                 const char *path, char *dst, size_t dst_size);
 
-//int newhttp(char* iadr,char* err=NULL);
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);
 /* Like newhttp(), but connect to the addr_index-th resolved address of the host

src/htsmodules.c

@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
 
 /* >>> Put all modules variables here */
 
-#if 0
-t_gzopen gzopen = NULL;
-t_gzread gzread = NULL;
-t_gzclose gzclose = NULL;
-#endif
-
 int V6_is_available = HTS_INET6;
 
 static char WHAT_is_available[64] = "";
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
   return -1;
 }
 
-//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
-//  while(chain->next != NULL) {
-//    chain = chain->next;
-//  }
-//  chain->next = calloct(1, sizeof(htscallbacks));
-//  assertf(chain->next != NULL);
-//  chain = chain->next;
-//  memset(chain, 0, sizeof(*chain));
-//  chain->exitFnc = exitFnc;
-//  chain->moduleHandle = moduleHandle;
-//}
-
 void clearCallbacks(htscallbacks * chain_);
 void clearCallbacks(htscallbacks * chain_) {
   htscallbacks *chain;

src/htsname.c

@@ -1332,8 +1332,7 @@ int url_savename(lien_adrfilsave *const afs,
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
-    if (*a != '.') {            // agh pas de point
+    if (*a != '.') {                         // agh pas de point
-      //strcatbuff(save,".none");                 // a éviter
       strcatbuff(afs->save, ".html");        // préférable!
       hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
                     adr_complete, fil_complete, afs->save);
@@ -1471,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
   fil_simplifie(afs->save);
 
   /* convert name to UTF-8 ? Note: already done while parsing. */
-  //if (charset != NULL && charset[0] != '\0') {
-  //  char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
-
-  //  if (s != NULL) {
-  //    hts_log_print(opt, LOG_DEBUG,
-  //                  "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
-  //                  save, s, charset);
-  //    strcpybuff(save, s);
-  //    free(s);
-  //  }
-  //}
 
   /* callback */
   RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1629,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
         int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
         int sameFil;
 
-        // NO - URL hack is only for stripping // and www.
-        //if (opt->urlhack != 0)
-        //  sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
-        //else
         sameFil = (strcmp(heap(i)->fil, normfil) == 0);
         if (sameAdr && sameFil) {       // ok c'est le même lien, adresse déja définie
           /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1698,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,
 
           strcpybuff(afs->save, tempo);
 
-          //printf("switched: %s\n",save);
-
         }                       // if
       }
 #if DEBUG_SAVENAME
       printf("\nEnd search, %s\n", fil_complete);
 #endif
-    } while(!nom_ok);
+    } while (!nom_ok);
-
   }
-  //printf("'%s' %s %s\n",save,adr,fil);
 
   return 0;
 }

src/htsopt.h

@@ -105,7 +105,6 @@ typedef struct htsfilters htsfilters;
 struct htsfilters {
   char ***filters; /**< pointer to the +/-pattern filter array */
   int *filptr;     /**< pointer to the current filter count */
-  // int*    filter_max;
 };
 
 /* User callbacks chain */
@@ -440,7 +439,6 @@ struct httrackp {
   float maxconn;          /**< max connections per second */
   int waittime;           /**< scheduled start time (wall-clock seconds) */
   hts_cachemode cache;    /**< cache generation mode */
-  // int aff_progress;     // progress bar
   hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
   t_proxy proxy;     /**< proxy configuration */
   hts_savename_83
@@ -492,7 +490,6 @@ struct httrackp {
   hts_verbosedisplay verbosedisplay; /**< animated text progress display */
   String footer; /**< footer/info line injected into pages */
   int maxcache;  /**< in-memory cache backing limit (bytes) */
-  // int maxcache_anticipate; // maximum links to anticipate (upper bound)
   hts_boolean ftp_proxy;    /**< use the HTTP proxy for FTP too */
   String filelist;          /**< file listing URLs to include */
   String urllist;           /**< file listing filters to include */
@@ -549,8 +546,6 @@ typedef struct hts_stat_struct hts_stat_struct;
 struct hts_stat_struct {
   LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
   LLint stat_bytes;     /**< total bytes written to disk */
-  // int HTS_TOTAL_RECV_STATE;  // status: 0 ok 1: slow down a little 2: slow
-  // down 3: a lot
   TStamp stat_timestart; /**< mirror start time */
   //
   LLint total_packed;    /**< compressed bytes received (on the wire) */
@@ -682,7 +677,6 @@ struct lien_url {
   int depth;        /**< remaining allowed depth; >0 strong, 0 weak */
   int pass2;        /**< second-pass marker; -1 means handled in background */
   char link_import; /**< imported after a move; skip the usual up/down rules */
-  // int moved;          // pointer to moved
   int retry;    /**< remaining retries */
   int testmode; /**< test only: send just a HEAD */
 };
@@ -727,9 +721,6 @@ struct lien_back {
   LLint chunk_size; /**< size of the chunk being loaded */
   LLint chunk_blocksize; /**< data size declared by the chunk */
   LLint compressed_size; /**< compressed size (stats only) */
-  //
-  // int links_index;        // to access liens[links_index]
-  //
   char info[256]; /**< status text, e.g. for FTP */
   int stop_ftp;   /**< stop flag for FTP */
   int finalized;  /**< finalized (memory optimization) */

src/htsparse.c

@@ -192,6 +192,48 @@ static void hts_automate_lookup(const script_automate *aut) {
   }
 }
 
+/* Attribute name owning the quoted value at 'quote' inside a tag, spanning
+   [name, *nend); NULL when the quote is not an attribute value. */
+static const char *dirty_attr_name(const char *quote, const char *tag_start,
+                                   const char **nend) {
+  const char *a = quote - 1;
+  while (a > tag_start && is_taborspace(*a))
+    a--;
+  if (a == tag_start || *a != '=')
+    return NULL;
+  a--;
+  while (a > tag_start && is_taborspace(*a))
+    a--;
+  *nend = a + 1;
+  while (a > tag_start && *a != '=' && *a != '\"' && *a != '\'' &&
+         !is_realspace(*a))
+    a--;
+  a++;
+  // a name starting right after '<' is the tag name, not an attribute
+  return a < *nend && a > tag_start + 1 ? a : NULL;
+}
+
+/* Accept the in-tag quoted value at 'quote' for dirty parsing? Resolves the
+   owning attribute itself (intag_startattr is unreliable mid-tag) and rejects
+   no-detect/xmlns names. */
+static hts_boolean dirty_attr_detectable(const char *quote,
+                                         const char *tag_start) {
+  const char *nend;
+  const char *name = dirty_attr_name(quote, tag_start, &nend);
+  int i;
+  if (name == NULL)
+    return HTS_FALSE;
+  for (i = 0; strnotempty(hts_nodetect[i]); i++) {
+    const int l = strfield(name, hts_nodetect[i]);
+    if (l && name + l == nend)
+      return HTS_FALSE;
+  }
+  i = strfield(name, "xmlns");
+  if (i && (name + i == nend || name[i] == ':'))
+    return HTS_FALSE;
+  return HTS_TRUE;
+}
+
 /* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
 static void hts_automate_increment(const script_automate *aut, int steps) {
   while (steps > 0) {
@@ -353,16 +395,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
       // terminaison (" ou ') du "<body onLoad=.."
       int inscriptgen = 0;      // on est dans un code générant, ex après obj.write("..
 
-      //int inscript_check_comments=0, inscript_in_comments=0;    // javascript comments
       char scriptgen_q = '\0';  // caractère faisant office de guillemet (' ou ")
 
-      //int no_esc_utf=0;      // ne pas echapper chars > 127
       int nofollow = 0;         // ne pas scanner
 
       //
       int parseall_lastc = '\0';        // dernier caractère parsé pour parseall
 
-      //int parseall_incomment=0;   // dans un /* */ (exemple: a = /* URL */ "img.gif";)
       //
       const char *intag_start = html;
       const char *intag_name = NULL;
@@ -475,12 +514,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           lastsaved = html;
         }
       }
-      // Detect UTF8 format
-      //if (is_unicode_utf8(r->adr, (unsigned int) r->size) == 1) {
-      //  no_esc_utf=1;
-      //} else {
-      //  no_esc_utf=0;
-      //}
 
       // Hack to prevent any problems with ram files of other files
       *(r->adr + r->size) = '\0';
@@ -645,8 +678,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           ) {
           intag = 1;
           intag_ctype = 0;
-          //parseall_incomment=0;
-          //inquote=0;  // effacer quote
           intag_start = html;
           for(intag_name = html + 1; is_realspace(*intag_name); intag_name++) ;
           intag_start_valid = 1;
@@ -664,8 +695,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               if (len > 0) {
                 if (strfield(token, "content-type")) {
                   intag_ctype = 1;
-                  //NOPE-we do not convert the whole page actually
-                  //intag_start[1] = 'X';
                 } else if (strfield(token, "refresh")) {
                   intag_ctype = 2;
                 }
@@ -726,7 +755,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil)), gmttime,
                           HTTRACK_VERSIONID, /* EOF */ NULL);
                   strcatbuff(tempo, eol);
-                  //fwrite(tempo,1,strlen(tempo),fp);
                   HT_ADD(tempo);
                 }
                 // Emit charset ?
@@ -754,6 +782,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           ) {
           if (inscript_tag) {
             inscript_tag = inscript = 0;
+            // reset the automaton on exit or its state leaks into plain HTML
+            inscript_state_pos = INSCRIPT_START;
             intag = 0;
             incomment = 0;
             intag_start_valid = 0;
@@ -765,8 +795,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             intag = 0;          //inquote=0;
 
             // entrée dans du javascript?
-            // on parse ICI car il se peut qu'on ait eu a parser les src=.. dedans
+            // on parse ICI car il se peut qu'on ait eu a parser les src=..
-            //if (!inscript) {  // sinon on est dans un obj.write("..
+            // dedans
             if ((intag_start_valid) && (check_tag(intag_start, "script")
                                         || check_tag(intag_start, "style")
                 )
@@ -812,12 +842,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             }
 #endif
           }
-          //}
+        } else if (intag || inscript ||
-        }
+                   in_media) { // nous sommes dans un tag/commentaire, tester si
-        //else if (*adr==34) {
+                               // on recoit un tag
-        //  inquote=(inquote?0:1);
-        //}
-        else if (intag || inscript || in_media) {       // nous sommes dans un tag/commentaire, tester si on recoit un tag
           int p_type = 0;
           int p_nocatch = 0;
           int p_searchMETAURL = 0;      // chercher ..URL=<url>
@@ -837,6 +864,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 if (*html == inscript_tag_lastc) {
                   /* sortir */
                   inscript_tag = inscript = 0;
+                  inscript_state_pos = INSCRIPT_START;
                   incomment = 0;
                   if (opt->parsedebug) {
                     HT_ADD("<@@ /inscript @@>");
@@ -1055,27 +1083,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       if (p) {
                         if (intag_ctype == 1) {
                           p = 0;
-#if 0
-                          //if ((pos=rech_tageq(html, "content"))) {
-                          char temp[256];
-                          char *token = NULL;
-                          int len = rech_endtoken(html + pos, &token);
-
-                          if (len > 0 && len < sizeof(temp) - 2) {
-                            char *chpos;
-
-                            temp[0] = '\0';
-                            strncat(temp, token, len);
-                            if ((chpos = strstr(temp, "charset"))
-                                && (chpos = strchr(chpos, '='))
-                              ) {
-                              chpos++;
-                              while(is_space(*chpos))
-                                chpod++;
-                              //chpos
-                            }
-                          }
-#endif
                         }
                         // <META HTTP-EQUIV="Refresh" CONTENT="3;URL=http://www.example.com">
                         else if (intag_ctype == 2) {
@@ -1260,21 +1267,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   }
                 } else if (inscript) {
 
-#if 0
-                  /* Check // javascript comments */
-                  if (*html == 10 || *html == 13) {
-                    inscript_check_comments = 1;
-                    inscript_in_comments = 0;
-                  } else if (inscript_check_comments) {
-                    if (!is_realspace(*html)) {
-                      inscript_check_comments = 0;
-                      if (html[0] == '/' && html[1] == '/') {
-                        inscript_in_comments = 1;
-                      }
-                    }
-                  }
-#endif
-
                   /* Parse */
                   assertf(inscript_name != NULL);
                   if (*html == '/'
@@ -1287,18 +1279,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       && inscript_locked == 0) {
                     const char *a = html;
 
-                    //while(is_realspace(*(--a)));
                     while(is_realspace(*a))
                       a--;
                     a--;
                     if (*a == '<') {    // sûr que c'est un tag?
                       inscript = 0;
+                      inscript_state_pos = INSCRIPT_START;
                       if (opt->parsedebug) {
                         HT_ADD("<@@ /inscript @@>");
                       }
                     }
-                  } else if (inscript_state_pos ==
+                  } else if (inscript_state_pos == INSCRIPT_START) {
-                             INSCRIPT_START /*!inscript_in_comments */ ) {
                     /*
                        Script Analyzing - different types supported:
                        foo="url"
@@ -1487,12 +1478,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               }
 
             } else {            // ptr == 0
-              //p=rech_tageq(adr,"primary");    // lien primaire, yeah
               p = 0;            // No stupid tag anymore, raw link
               valid_p = 1;      // Valid even if p==0
               while((html[p] == '\r') || (html[p] == '\n'))
                 p++;
-              //can_avoid_quotes=1;
               ending_p = '\r';
             }
 
@@ -1505,25 +1494,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           // risque: générer de faux fichiers parazites
           // fix: ne parse plus dans les commentaires
           // ------------------------------------------------------------
-          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 && (ptr > 0) && (!in_media) /* && (!inscript_in_comments) */ ) {  // option parsing "brut"
+          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 &&
-            //int incomment_justquit=0;
+              (ptr > 0) && (!in_media)) { // option parsing "brut"
             if (!is_realspace(*html)) {
               int noparse = 0;
 
-              // Gestion des /* */
-#if 0
-              if (inscript) {
-                if (parseall_incomment) {
-                  if ((*html == '/') && (*(html - 1) == '*'))
-                    parseall_incomment = 0;
-                  incomment_justquit = 1;       // ne pas noter dernier caractère
-                } else {
-                  if ((*html == '/') && (*(html + 1) == '*'))
-                    parseall_incomment = 1;
-                }
-              } else
-                parseall_incomment = 0;
-#endif
               /* ensure automate state  0 (not in comments, quotes..) */
               if (inscript
                   && (inscript_state_pos != INSCRIPT_INQUOTE
@@ -1537,7 +1512,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
               // recherche d'URLs
               if (!noparse) {
-                //if ((!parseall_incomment) && (!noparse)) {
                 if (!p) {       // non déja trouvé
                   if (html != r->adr) {  // >1 caractère
                     // scanner les chaines
@@ -1561,15 +1535,16 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             && (count > 0)) {
                           char c;
 
-                          //char* aend;
-                          //
-                          //aend=a;     // sauver début
                           a++;
                           while(is_taborspace(*a))
                             a++;
                           c = *a;
-                          if (strchr("),;>/+\r\n", c)) {        // exemple: ..img.gif";
+                          // in-tag, an attribute value ends at its quote: no
-                            // le / est pour funct("img.gif" /* URL */);
+                          // delimiter required after it (mid-tag attrs, #201)
+                          if (strchr("),;>/+\r\n", c) ||
+                              (intag && !inscript && intag_start_valid &&
+                               dirty_attr_detectable(html, intag_start))) {
+                            // '/' covers a value followed by a JS comment
                             char BIGSTK tempo[HTS_URLMAXSIZE * 2];
                             char type[256];
                             int url_ok = 0;     // url valide?
@@ -1614,10 +1589,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               if (!invalid_url) {
                                 // Un plus à la fin? Alors ne pas prendre sauf si extension ("/toto.html#"+tag)
                                 if (c != '+') { // PAS de plus à la fin
-#if 0
+                                  // "Comparisons of scheme names MUST be
-                                  char *a;
+                                  // case-insensitive" (RFC2616)
-#endif
-                                  // "Comparisons of scheme names MUST be case-insensitive" (RFC2616)                                  
                                   if ((strfield(tempo, "http:"))
                                       || (strfield(tempo, "ftp:"))
 #if HTS_USEOPENSSL
@@ -1630,15 +1603,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                     if (inscript)       // sinon si pas javascript, méfiance (répertoire style base?)
                                       url_ok = 1;
                                   }
-#if 0
-                                  else if ((a = strchr(tempo, '/'))) {  // un slash: ok..
-                                    if (inscript) {     // sinon si pas javascript, méfiance (style "text/css")
-                                      if (strchr(a + 1, '/'))   // un seul / : abandon (STYLE type='text/css')
-                                        if (!strchr(tempo, ' '))        // avoid spaces (too dangerous for comments)
-                                          url_ok = 1;
-                                    }
-                                  }
-#endif
                                 }
                                 // Prendre si extension reconnue
                                 if (!url_ok) {
@@ -1717,9 +1681,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                     }
                   }
-                }               // p == 0               
+                } // p == 0
 
-              }                 // not in comment
+              } // not in comment
 
               // plus dans un commentaire
               if (inscript_state_pos == INSCRIPT_START
@@ -1728,17 +1692,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               }
 
             }                   // if realspace
-          }                     // if parseall
+          } // if parseall
 
           // ------------------------------------------------------------
           // p!=0 : on a repéré un éventuel lien
           // ------------------------------------------------------------
           //
-          if ((p > 0) || (valid_p)) {   // on a repéré un lien
+          if ((p > 0) || (valid_p)) { // on a repéré un lien
-            //int lien_valide=0;
             const char *eadr = NULL;  /* fin de l'URL */
 
-            //char* quote_adr=NULL;     /* adresse du ? dans l'adresse */
             int ok = 1;
             char quote = '\0';
             int quoteinscript = 0;
@@ -1761,7 +1723,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               lastsaved = html;  // dernier écrit+1
             }
             // sauter espaces
-            // adr+=p;
             hts_automate_increment(&saut, p);
             while((is_space(*html)
                    || (inscriptgen && html[0] == '\\' && is_space(html[1])
@@ -1790,8 +1751,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             // sauter éventuel \" ou \' javascript
             if (inscript) {     // on est dans un obj.write("..
               if (*html == '\\') {
-                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) {      // \" ou \'
+                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
-                  // html+=2;    // sauter
                   hts_automate_increment(&saut, 2);
                 }
               }
@@ -1866,7 +1826,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   ok = -1;      // ne pas traiter ce lien
 
                 if (ok > 0) {
-                  //if (*eadr!=' ') {  
                   if (is_space(*eadr)) {        // guillemets,CR, etc
                     if ((*eadr == quote && (!quoteinscript || *(eadr - 1) == '\\'))     // end quote
                         || (noquote && (*eadr == '\"' || *eadr == '\''))        // end at any quote
@@ -1912,7 +1871,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       break;
                     }
                   }
-                  //}
                 }
                 eadr++;
               } while(ok == 1);
@@ -1941,14 +1899,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               char BIGSTK lien[HTS_URLMAXSIZE * 2];
               int meme_adresse = 0;     // 0 par défaut pour primary
 
-              //char *copie_de_adr=html;
-              //char* p;
-
               // construire lien (découpage)
               if (eadr - html - 1 < HTS_URLMAXSIZE) {        // pas trop long?
                 strncpy(lien, html, eadr - html - 1);
                 lien[eadr - html - 1] = '\0';
-                //printf("link: %s\n",lien);          
                 // supprimer les espaces
                 while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
                   lien[strlen(lien) - 1] = '\0';
@@ -2192,7 +2146,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     a += 2;
                   else
                     a = lien;
-                  // while((*a) && (*a!='/') && (*a!=':')) a++;
                   a = jump_toport(a);
                   if (a) {      // port
                     int port = 0;
@@ -2200,9 +2153,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     char *b = a + 1;
 
 #if HTS_USEOPENSSL
-                    // FIXME
-                    //if (strfield(adr, "https:")) {
-                    //}
 #endif
                     while(isdigit((unsigned char) *b)) {
                       port *= 10;
@@ -2293,13 +2243,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             *(a + 1) = '\0';
                           }
                         }
-                        //char BIGSTK tempo[HTS_URLMAXSIZE*2];
-                        //strcpybuff(tempo,"http://");
-                        //strcatbuff(tempo,urladr());    // host
-                        //if (*lien!='/')
-                        //  strcatbuff(tempo,"/");
-                        //strcatbuff(tempo,lien);
-                        //strcpybuff(lien,tempo);
                       }
                     }
 
@@ -2328,22 +2271,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                       // stocker base ou codebase?
                       switch (p_type) {
-                      case 2:{
+                      case 2: {
-                          //if (*lien!='/') strcatbuff(base,"/");
+                        strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
-                          strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
+                      } break; // base
-                        }
+                      case -2: {
-                        break;  // base
+                        strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
-                      case -2:{
+                      } break; // base
-                          //if (*lien!='/') strcatbuff(codebase,"/");
-                          strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
-                        }
-                        break;  // base
                       }
 
                       hts_log_print(opt, LOG_DEBUG,
                                     "code/codebase link %s base %s", lien,
                                     base);
-                      //printf("base code: %s - %s\n",lien,base);
                     }
 
                   } else {
@@ -2456,7 +2394,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 // Tester si un lien doit être accepté ou refusé (wizard)
                 // forbidden_url=1 : lien refusé
                 // forbidden_url=0 : lien accepté
-                //if ((ptr>0) && (p_type!=2) && (p_type!=-2)) {    // tester autorisations?
                 if ((p_type != 2) && (p_type != -2)) {  // tester autorisations?
                   if (!p_nocatch) {
                     if (afs.af.adr[0] != '\0') {
@@ -2506,9 +2443,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                       /* Calc */
                       last_adr[0] = '\0';
-                      //char last_fil[HTS_URLMAXSIZE*2]="";
+                      strcpybuff(last_adr, afs.af.adr); // ancienne adresse
-                      strcpybuff(last_adr, afs.af.adr);        // ancienne adresse
-                      //strcpybuff(last_fil,fil);    // ancien chemin
                       r_sv =
                         url_savename(&afs, &former, heap(ptr)->adr, heap(ptr)->fil, opt,
                                      sback, cache, hash, ptr,
@@ -2539,7 +2474,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             }
                           }
                         }
-                        //import_done=1;    // c'est un import!
                         meme_adresse = 0;       // on a changé
                       }
                     } else {
@@ -2737,11 +2671,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               if (lienrelatif(tempo, sizeof(tempo), save,
                                               relativesavename()) == 0) {
                                 /* Never escape high-chars (we don't know the encoding!!) */
-                                inplace_escape_uri_utf(tempo, sizeof(tempo));  // escape with %xx
+                                inplace_escape_uri_utf(
-                                //if (!no_esc_utf)
+                                    tempo, sizeof(tempo)); // escape with %xx
-                                //  escape_uri(tempo);     // escape with %xx
-                                //else
-                                //  escape_uri_utf(tempo);     // escape with %xx
                                 HT_ADD_HTMLESCAPED(tempo);      // page externe
                                 if (add_url) {
                                   HT_ADD("?link=");     // page externe
@@ -2978,16 +2909,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           // unquoted url() (CSS/JS): keep parens escaped
                           if (ending_p == ')')
                             escape_url_parens(tempo, sizeof(tempo));
-
-                          //if (!no_esc_utf)
-                          //  escape_uri(tempo);     // escape with %xx
-                          //else {
-                          //  /* No escaping at all - remaining upper chars will be escaped below */
-                          //  /* FIXME - Should be done in all local cases */
-                          //  //x_escape_html(tempo);
-                          //  //escape_uri_utf(tempo);     // FIXME - escape with %xx
-                          //  //escape_uri(tempo);     // escape with %xx
-                          //}
                         }
                         hts_log_print(opt, LOG_DEBUG,
                                       "relative link at %s build with %s and %s: %s",
@@ -3045,22 +2966,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               HT_ADD(tempo4);   // refresh code="
                             }
                           }
-                          //lastsaved=adr;    // dernier écrit+1
                         }
 
                         if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
-                          // convert to local codepage - NOT, already converted into %NN, and passed to the remote server so we do not have anything to do
-                          //if (str->page_charset_ != NULL && *str->page_charset_ != '\0') {
-                          //  char *const local_save = hts_convertStringFromUTF8(tempo, strlen(tempo), str->page_charset_);
-                          //  if (local_save != NULL) {
-                          //    strcpybuff(tempo, local_save);
-                          //    free(local_save);
-                          //  } else {
-                          //    if ((opt->debug>1) && (opt->log!=NULL)) {
-                          //      fprintf(opt->log, "Warning: could not build local charset representation of '%s' in '%s'"LF, tempo, str->page_charset_);
-                          //    }
-                          //  }
-                          //}
 
                           // écrire le lien modifié, relatif
                           // Note: escape all chars, even >127 (no UTF)
@@ -3084,16 +2992,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                     } // sinon le lien sera écrit normalement
 
-#if 0
-                    if (fexist(save)) { // le fichier existe..
-                      adr[0] = '\0';
-                      //if ((opt->debug>0) && (opt->log!=NULL)) {
-                      hts_log_print(opt, LOG_WARNING,
-                                    "Link has already been written on disk, cancelled: %s",
-                                    save);
-                    }
-#endif
-
                     /* Security check */
                     if (strlen(afs.save) >= HTS_URLMAXSIZE) {
                       afs.af.adr[0] = '\0';
@@ -3135,9 +3033,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         /*
                            if (strnotempty(save)) {
                            if (ishtml(opt,save) == 1) {
-                           // descore_prio = 2;
                            } else {
-                           // descore_prio = 1;
                            }
                            }
                          */
@@ -3173,8 +3069,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           // >>>> CREER LE LIEN <<<<
                           //
                           // enregistrer lien à charger
-                          //heap_top()->adr[0]=heap_top()->fil[0]=heap_top()->sav[0]='\0';
-                          // même adresse: l'objet père est l'objet père de l'actuel
 
                           // DEBUT ROBOTS.TXT AJOUT
                           if (!just_test_it) {
@@ -3253,7 +3147,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             heap_top()->premier = heap(ptr)->premier;
                           else  // sinon l'objet père est le précédent lui même
                             heap_top()->premier = heap_top_index();
-                          // heap_top()->premier=ptr;
 
                           heap_top()->precedent = ptr;
                           // noter la priorité
@@ -3265,9 +3158,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           heap_top()->pass2 = pass_fix;
                           heap_top()->retry = opt->retry;
 
-                          //strcpybuff(heap_top()->adr,adr);
-                          //strcpybuff(heap_top()->fil,fil);
-                          //strcpybuff(heap_top()->sav,save); 
                           if (!just_test_it) {
                             hts_log_print(opt, LOG_DEBUG,
                                           "OK, NOTE: %s%s -> %s",
@@ -3302,7 +3192,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             if (eadr > html) {
               hts_automate_increment(&saut, (int) (eadr - 1 - html));
             }
-            // adr=eadr-1;  // ** sauter
 
             /* srcset candidate loop: skip the descriptor and comma, then
                re-enter the capture for the next URL. Backward goto, not a loop:
@@ -3332,9 +3221,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                inscript_state_pos = INSCRIPT_START;
                } */
 
-          }                     // if (p) 
+ } // if (p)
 
-        }                       // si '<' ou '>'
+        } // si '<' ou '>'
 
         // plus loin
         html++;                  // automate will be checked next loop
@@ -3404,9 +3293,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             *stre->exit_xh_ = 1;        // exit requested
             XH_uninit;
             return -1;
-            //adr = r->adr + r->size;  // exit
           } else if (opt->state._hts_cancel == 1) {
-            // adr = r->adr + r->size;  // exit
             nofollow = 1;       // moins violent
             opt->state._hts_cancel = 0;
           }
@@ -3464,9 +3351,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
         fp = NULL;
       }
     }
-    // sauver fichier
-    //structcheck(savename());
-    //filesave(opt,r->adr,r->size,savename());
 
   }                             // analyse OK
 
@@ -3505,17 +3389,11 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
   // DEBUT rattrapage des 301,302,307..
   // ------------------------------------------------------------
   if (!error) {
-    ////////{
-    // on a chargé un fichier en plus
-    // if (!error) stat_loaded+=r.size;
 
     // ------------------------------------------------------------
     // Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing 
     // ------------------------------------------------------------
     if (HTTP_IS_REDIRECT(r->statuscode)) {
-      //if (r->adr!=NULL) {   // adr==null si fichier direct. [catch: davename normalement si cgi]
-      //int i=0;
-      // char* p;
 
       hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, urladr(), urlfil());
 
@@ -3542,7 +3420,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
           // A same-file alias redirect must be followed, not stubbed (#159).
           const hts_boolean same_savefile = hts_redirect_same_savefile(
               opt, urladr(), urlfil(), moved->adr, moved->fil);
-          //if (ident_url_absolute(mov_url,moved->adr,moved->fil)!=-1) {    // ok URL reconnue
           // c'est (en gros) la même URL..
           // si c'est un problème de casse dans le host c'est que le serveur est buggé
           // ("RFC says.." : host name IS case insensitive)
@@ -3582,7 +3459,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
             } /* sinon traité normalement */
           }
 
-          //if ((strfield2(moved->adr,urladr())!=0) && (strfield2(moved->fil,urlfil())!=0)) {  // identique à casse près
           if (get_it == 1) {
             // court-circuiter le reste du traitement
             // et reculer pour mieux sauter
@@ -3593,9 +3469,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
             error = 1;
             hts_invalidate_link(opt, ptr);  // invalidate hashtable entry
             // noter NOUVEAU lien
-            //xxc xxc
-            //  set_prio_to=0+1;  // protection if the moved URL is an html page!!
-            //xxc xxc
             {
               // calculer lien et éventuellement modifier addresse/fichier
               if (url_savename(&savedmoved, NULL,
@@ -3633,8 +3506,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
               }
             }
 
-            //printf("-> %s %s %s\n",liens[lien_tot-1]->adr,liens[lien_tot-1]->fil,liens[lien_tot-1]->sav);
-
             // note métaphysique: il se peut qu'il y ait un index.html et un INDEX.HTML
             // sous DOS ca marche pas très bien... mais comme je suis génial url_savename()
             // est à même de régler ce problème
@@ -3756,7 +3627,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
 
       // cas où l'on peut reessayer
       switch (r->statuscode) {
-        //case -1: can_retry=1; break;
       case STATUSCODE_TIMEOUT:
         if (opt->hostcontrol) { // timeout et retry épuisés
           if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) &&
@@ -3888,7 +3758,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
     // FIN rattrapage des 301,302,307..
     // ------------------------------------------------------------
 
-  }                             // if !error
+  } // if !error
 
   /* Apply changes */
   ENGINE_SAVE_CONTEXT();
@@ -3955,9 +3825,6 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
           }
         }
       }
-      // On désalloue le buffer d'enregistrement des chemins créée, au cas où pendant la pause
-      // l'utilisateur ferait un rm -r après avoir effectué un tar
-      // structcheck_init(1);
       {
         FILE *fp =
           fopen(fconcat
@@ -4338,33 +4205,8 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
                     "link #%d is ready, no more on the stack, skipping: %s%s..",
                     ptr, urladr(), urlfil());
 
-      // prochain lien
+      return 2; // goto jump_if_done;
-      // ptr++;
-
-      return 2;                 // goto jump_if_done;
-
     }
-#if 0
-    /* FIXME - finalized HAS NO MORE THIS MEANING */
-    /* link put in cache by the backing system for memory spare - reclaim */
-    else if (back[b].finalized) {
-      assertf(back[b].r.adr == NULL);
-      /* read file in cache */
-      back[b].r =
-        cache_read_ro(opt, cache, back[b].url_adr, back[b].url_fil,
-                      back[b].url_sav, back[b].location_buffer);
-      /* ensure correct location buffer set */
-      back[b].r.location = back[b].location_buffer;
-      if (back[b].r.statuscode == STATUSCODE_INVALID) {
-        hts_log_print(opt, LOG_ERROR,
-                      "Unexpected error: %s%s not found anymore in cache",
-                      back[b].url_adr, back[b].url_fil);
-      } else {
-        hts_log_print(opt, LOG_DEBUG, "reclaim file %s%s (%d)", back[b].url_adr,
-                      back[b].url_fil, back[b].r.statuscode);
-      }
-    }
-#endif
 
     if (!opt->verbosedisplay) {
       if (!opt->quiet) {
@@ -4388,7 +4230,6 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
       }
       fflush(stdout);
     }
-    //}
 
     // ------------------------------------------------------------
     // Vérificateur d'intégrité
@@ -4556,9 +4397,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) {
           lien_back delayed_back;
 
-          //char BIGSTK delayed_ctype[128];
-          // delayed_ctype[0] = '\0';
-          // strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
           back_copy_static(&back[b], &delayed_back);
 
           /* Delete entry */
@@ -4566,6 +4404,12 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           back_maydelete(opt, cache, sback, b);    // cancel
           b = -1;
 
+          /* the cancel may leave the now-unreferenced placeholder on disk
+           * (#483) */
+          if (fexist_utf8(delayed_back.url_sav)) {
+            back_delayed_discard(opt, &delayed_back);
+          }
+
           /* Recompute filename with MIME type */
           afs->save[0] = '\0';
           url_savename(afs, former, heap(ptr)->adr,
@@ -4657,9 +4501,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         if (b >= 0) {
           lien_back delayed_back;
 
-          //char BIGSTK delayed_ctype[128];
-          //delayed_ctype[0] = '\0';
-          //strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
           back_copy_static(&back[b], &delayed_back);
 
           /* Error */
@@ -4782,6 +4623,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
 
           /* Still have a back reference */
           if (b >= 0) {
+            /* move a still-writing placeholder before the url_sav patch
+               blinds every cleanup to it (#483) */
+            back_delayed_rename(opt, &back[b], afs->save);
             /* patch url_sav BEFORE finalize: it records/caches under this name
              */
             strcpybuff(back[b].url_sav, afs->save);
@@ -4808,13 +4652,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       /* 'no error page' selected or file discarded by size rules! */
       if (!opt->errpage || (in_error == STATUSCODE_TOO_BIG)) {
         /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
-#if 0
+        /* Do not post-exclude the link here (*forbidden_url): the cache system
-        /* No (3.43) - don't do that. We must not post-exclude an authorized link, because this will prevent the cache
+           would never process it again, and it would be refetched endlessly */
-           system from processing it, leading to refetch it endlessly. Just accept it, and handle the error as
-           usual during parsing.
-         */
-        *forbidden_url = 1;     /* Forbidden! */
-#endif
         if (in_error == STATUSCODE_TOO_BIG) {
           hts_log_print(opt, LOG_INFO,
                         "link not taken because of its size (%d bytes) at %s%s",

src/htsparse.h

@@ -66,7 +66,6 @@ struct htsmoduleStructExtended {
   char ***filters_;
   robots_wizard *robots_;
   hash_struct *hash_;
-  //int *lien_max_;
 
   /* Base & codebase */
   char *base;

src/htsselftest.c

@@ -1347,6 +1347,30 @@ static int st_cache_writefail(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
+  int err;
+
+  if (argc < 1) {
+    fprintf(stderr, "cache-corrupt: needs a directory\n");
+    return 1;
+  }
+  err = cache_corruption_selftest(opt, argv[0]);
+  printf("cache-corrupt: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
+static int st_reconcile(httrackp *opt, int argc, char **argv) {
+  int err;
+
+  if (argc < 1) {
+    fprintf(stderr, "reconcile: needs a directory\n");
+    return 1;
+  }
+  err = cache_reconcile_selftest(opt, argv[0]);
+  printf("cache-reconcile: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 static int st_dns(httrackp *opt, int argc, char **argv) {
   const int err = dns_selftests(opt);
 
@@ -2119,6 +2143,10 @@ static const struct selftest_entry {
      st_cache_golden},
     {"cache-writefail", "<dir>", "cache write-failure handling self-test",
      st_cache_writefail},
+    {"reconcile", "<dir>", "cache generation reconcile policy self-test",
+     st_reconcile},
+    {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
+     st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
     {"cookies", "", "cookie request-header self-test", st_cookies},
     {"useragent", "", "default User-Agent self-test", st_useragent},

src/htsserver.c

@@ -265,7 +265,6 @@ T_SOC smallserver_init(int *port, char *adr) {
       SOCaddr_initport(server, *port);
       if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
         if (listen(soc, 10) >= 0) {
-          // SOCaddr_inetntoa(adr, 128, server2);
           strcpy(adr, h_loc);
         } else {
 #ifdef _WIN32
@@ -439,7 +438,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
         meth = 10;
       } else {
 #ifdef _DEBUG
-        // assert(FALSE);
 #endif
       }
       if (meth) {
@@ -1360,7 +1358,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
 
             StringCat(headers, error_hdr);
             StringCat(output, error);
-            //assert(file == NULL);
           }
         }
       } else {
@@ -1388,12 +1385,10 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                   != StringLength(output)))
         ) {
 #ifdef _DEBUG
-        //assert(FALSE);
 #endif
       }
     } else {
 #ifdef _DEBUG
-      // assert(FALSE);
 #endif
     }
 
@@ -1540,8 +1535,7 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
             } while(strnotempty(test));
           }
 
-          if (!strnotempty(test)) {     // éviter doublons
+          if (!strnotempty(test)) { // éviter doublons
-            // conv_printf(key,key);
             const size_t len = strlen(intkey);
             char *const buff = (char *) malloc(len + 1);
 
@@ -1632,8 +1626,6 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
                     intkey = "";
                 } else {
                   if (loops > 0) {
-                    //err_msg += intkey;
-                    //err_msg += " ";
                   }
                 }
               }
@@ -1740,13 +1732,7 @@ static void LANG_DELETE(void) {
 }
 
 // sélection de la langue
-static void LANG_INIT(const char *path) {
+static void LANG_INIT(const char *path) { LANG_T(path, 0); }
-  //CWinApp* pApp = AfxGetApp();
-  //if (pApp) {
-  /* pApp->GetProfileInt("Language","IntId",0); */
-  LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
-  //}
-}
 
 static int LANG_T(const char *path, int l) {
   if (l >= 0) {

src/htstools.c

@@ -68,16 +68,6 @@ struct find_handle_struct {
   char path[2048];
 };
 #endif
-//#ifndef HTS_DEF_FWSTRUCT_topindex_chain
-//#define HTS_DEF_FWSTRUCT_topindex_chain
-//typedef struct topindex_chain topindex_chain;
-//#endif
-//struct topindex_chain {
-//  int level;                    /* sort level */
-//  char *category;               /* category */
-//  char name[2048];              /* path */
-//  struct topindex_chain *next;  /* next element */
-//};
 
 /* Tools */
 
@@ -293,7 +283,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2];
   char *curr;
 
-  //int n=0;
   char *a;
   int slash = 0;
 
@@ -338,7 +327,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
     if (*curr == '/')
       curr++;
     l = link;
-    //c=curr;
     // couper ce qui est commun
     while((streql(*link, *curr)) && (*link != 0)) {
       link++;
@@ -350,8 +338,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
       link--;
       curr--;
     }
-    //if (*link=='/') link++;
-    //if (*curr=='/') curr++;
   }
 
   // calculer la profondeur du répertoire courant et remonter
@@ -362,7 +348,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   while(*a)
     if (*(a++) == '/')
       strlcatbuff(s, "../", ssize);
-  //if (strlen(s)==0) strcatbuff(s,"/");
 
   if (slash)
     strlcatbuff(s, "/", ssize); // keep it absolute!

src/htsweb.c

@@ -639,8 +639,7 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
           for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) {        // no lien
             int i = (back_index + _i) % back_max;       // commencer par le "premier" (l'actuel)
 
-            if (back[i].status >= 0) {  // signifie "lien actif"
+            if (back[i].status >= 0) { // signifie "lien actif"
-              // int ok=0;  // OPTI
               ok = 0;
               switch (j) {
               case 0:          // prioritaire

src/htswizard.c

@@ -188,12 +188,10 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
   /* Doit-on traiter les non html? */
   if ((opt->getmode & HTS_GETMODE_NONHTML) == 0) { // non on ne doit pas
-    if (!ishtml(opt, fil)) {    // non il ne faut pas
+    if (!ishtml(opt, fil)) {                       // non il ne faut pas
-      //adr[0]='\0';    // ne pas traiter ce lien, pas traiter
       forbidden_url = 1;        // interdire récupération du lien
       hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr,
                     fil);
-
     }
   }
 
@@ -352,7 +350,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
   } else {                      // adresse différente, sortir?
 
-    //if (!opt->wizard) {    // mode non wizard
     // doit-on traiter ce lien?.. vérifier droits de sortie
     switch ((opt->travel & HTS_TRAVEL_SCOPE_MASK)) {
     case HTS_TRAVEL_SAME_ADDRESS:
@@ -380,7 +377,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
       if ((i > 0) && (j > 0)) {
         if (!strfield2(adr + i, urladr() + j)) { // !=
           if (!opt->wizard) {                    // mode non wizard
-            // printf("refused: %s\n",adr);
             forbidden_url = 1; // pas même domaine
             hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s",
                           adr, fil);
@@ -408,7 +404,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
       if ((i > 0) && (j > 0)) {
         if (!strfield2(adr + i, urladr() + j)) { // !-
           if (!opt->wizard) {                    // mode non wizard
-            // printf("refused: %s\n",adr);
             forbidden_url = 1; // pas même .xx
             hts_log_print(opt, LOG_DEBUG,
                           "foreign location link canceled: %s%s", adr, fil);
@@ -439,7 +434,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
   // récupérer les liens à côtés d'un lien (nearlink) (nvelle pos)
   if (forbidden_url != 0 && opt->nearlink) {
     if (!ishtml(opt, fil)) {    // non html
-      //printf("ok %s%s\n",ad,fil);
       forbidden_url = 0;        // autoriser
       may_set_prio_to = 1 + 1;  // set prio to 1 (parse but skip urls) if near is the winner
       hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil);
@@ -887,7 +881,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
         }
       }
     }
-    //adr[0]='\0';  // cancel
   }
   // -------------------- FINAL PHASE --------------------
   // Test if the "Near" test won

src/htszlib.h

@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com
 
 /* ZLib */
 #include "zlib.h"
-//#include "zutil.h"
 
 /* MiniZip */
 #include "minizip/zip.h"

src/httrack.c

@@ -54,7 +54,6 @@ static int linput(FILE * fp, char *s, int max);
 #include "htswrap.h"
 
 /* specific definitions */
-//#include "htsbase.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -454,8 +453,7 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b
           for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) {        // no lien
             int i = (back_index + _i) % back_max;       // commencer par le "premier" (l'actuel)
 
-            if (back[i].status >= 0) {  // signifie "lien actif"
+            if (back[i].status >= 0) { // signifie "lien actif"
-              // int ok=0;  // OPTI
               ok = 0;
               switch (j) {
               case 0:          // prioritaire
@@ -773,28 +771,7 @@ static void sig_finish(int code) {      // finir et quitter
   fprintf(stderr, "\nExit requested to engine (signal %d)\n", code);
 }
 
-#ifdef _WIN32
+#ifndef _WIN32
-#if 0
-static void sig_ask(int code) { // demander
-  char s[256];
-
-  signal(code, sig_term);       // quitter si encore
-  printf("\nQuit program/Interrupt/Cancel? (Q/I/C) ");
-  fflush(stdout);
-  scanf("%s", s);
-  if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
-      || (s[0] == 'q') || (s[0] == 'Q'))
-    exit(0);                    // quitter
-  else if ((s[0] == 'i') || (s[0] == 'I')) {
-    if (global_opt != NULL) {
-      // ask for stop
-      global_opt->state.stop = 1;
-    }
-  }
-  signal(code, sig_ask);        // remettre signal
-}
-#endif
-#else
 static void sig_doback(int blind);
 static void sig_back(int code) {        // ignorer et mettre en backing 
   if (global_opt != NULL && !global_opt->background_on_suspend) {
@@ -809,36 +786,6 @@ static void sig_back(int code) {        // ignorer et mettre en backing
   }
 }
 
-#if 0
-static void sig_ask(int code) { // demander
-  char s[256];
-
-  signal(code, sig_term);       // quitter si encore
-  printf
-    ("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) ");
-  fflush(stdout);
-  scanf("%s", s);
-  if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
-      || (s[0] == 'q') || (s[0] == 'Q'))
-    exit(0);                    // quitter
-  else if ((s[0] == 'b') || (s[0] == 'B') || (s[0] == 'a') || (s[0] == 'A'))
-    sig_doback(0);              // arrière plan
-  else if ((s[0] == 'l') || (s[0] == 'L'))
-    sig_doback(1);              // arrière plan
-  else if ((s[0] == 'i') || (s[0] == 'I')) {
-    if (global_opt != NULL) {
-      // ask for stop
-      printf("finishing pending transfers.. please wait\n");
-      global_opt->state.stop = 1;
-    }
-    signal(code, sig_ask);      // remettre signal
-  } else {
-    printf("cancel..\n");
-    signal(code, sig_ask);      // remettre signal
-  }
-}
-#endif
-
 static void sig_brpipe(int code) {      // treat if necessary
   signal(code, sig_brpipe);
 }
@@ -951,23 +898,12 @@ static void sig_leave(int code) {
 
 static void signal_handlers(void) {
 #ifdef _WIN32
-#if 0                           /* BUG366763 */
-  signal(SIGINT, sig_ask);      // ^C
-#else
   signal(SIGINT, sig_leave);    // ^C
-#endif
   signal(SIGTERM, sig_finish);  // kill <process>
 #else
-#if 0                           /* BUG366763 */
-  signal(SIGHUP, sig_back);     // close window
-#endif
   signal(SIGTSTP, sig_back);    // ^Z
   signal(SIGTERM, sig_finish);  // kill <process>
-#if 0                           /* BUG366763 */
-  signal(SIGINT, sig_ask);      // ^C
-#else
   signal(SIGINT, sig_leave);    // ^C
-#endif
   signal(SIGPIPE, sig_brpipe);  // broken pipe (write into non-opened socket)
   signal(SIGCHLD, sig_ignore);  // child change status
 #endif

src/proxy/proxytrack.c

@@ -250,13 +250,6 @@ static int gethost(const char *hostname, SOCaddr * server) {
     struct addrinfo hints;
 
     memset(&hints, 0, sizeof(hints));
-#if 0
-    if (IPV6_resolver == 1)     // V4 only (for bogus V6 entries)
-      hints.ai_family = PF_INET;
-    else if (IPV6_resolver == 2)        // V6 only (for testing V6 only)
-      hints.ai_family = PF_INET6;
-    else
-#endif
     hints.ai_family = PF_UNSPEC;
     hints.ai_socktype = SOCK_STREAM;
     hints.ai_protocol = IPPROTO_TCP;
@@ -340,12 +333,7 @@ int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort,
   T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM);
 
   if (soc != INVALID_SOCKET && socICP != INVALID_SOCKET) {
-    //char url[HTS_URLMAXSIZE * 2];
-    //char method[32];
-    //char data[32768];
 
-    //url[0] = method[0] = data[0] = '\0';
-    //
     printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort);
     printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort);
 #ifndef _WIN32
@@ -950,7 +938,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
     char *command;
     char *proto;
     char *surl;
-    //int directHit = 0;
     int headRequest = 0;
     int listRequest = 0;
 
@@ -1092,10 +1079,8 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
 
       /* Post-process request */
       if (link_has_authority(surl)) {
-        if (strncasecmp
+        if (strncasecmp(surl, "http://proxytrack/",
-            (surl, "http://proxytrack/",
+                        sizeof("http://proxytrack/") - 1) == 0) {
-             sizeof("http://proxytrack/") - 1) == 0) {
-          //directHit = 1;        /* Another direct hit hack */
         }
         StringCopy(url, surl);
       } else {
@@ -1116,7 +1101,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
               toHit += 7;
             }
             /* Direct hit */
-            //directHit = 1;
             StringCopy(url, "");
             if (!link_has_authority(toHit))
               StringCat(url, "http://");
@@ -1127,7 +1111,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
             const char *toHit = surl + sizeof("/proxytrack/") - 1;
 
             /* Direct hit */
-            //directHit = 1;
             StringCopy(url, "");
             if (!link_has_authority(toHit))
               StringCat(url, "http://");
@@ -1611,10 +1594,7 @@ static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) {
         unsigned char Opcode = buffer[0];
         unsigned char Version = buffer[1];
         unsigned short Message_Length = READ_NET16(&buffer[2]);
-        unsigned int Request_Number = READ_NET32(&buffer[4]);   /* Session ID */
+        unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
-        //unsigned int Options = READ_NET32(&buffer[8]);
-        //unsigned int Option_Data = READ_NET32(&buffer[12]);     /* ICP_FLAG_SRC_RTT */
-        //unsigned int Sender_Host_Address = READ_NET32(&buffer[16]);     /* ignored */
         unsigned char *Payload = &buffer[20];
 
         buffer[bufferSize] = '\0';      /* Ensure payload is NULL terminated */

src/proxy/store.c

@@ -1035,7 +1035,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
       if (unzOpenCurrentFile(index->zFile) == Z_OK) {
         char headerBuff[8192 + 2];
         int readSizeHeader;
-        //int totalHeader = 0;
         int dataincache = 0;
 
         /* For BIG comments */
@@ -1074,9 +1073,8 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
               ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified);      // last-modified
               ZIP_READFIELD_STRING(line, value, "Etag", r->etag);       // Etag
               ZIP_READFIELD_STRING(line, value, "Location", r->location);       // 'location' pour moved
-              ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo);      // Content-disposition
+              ZIP_READFIELD_STRING(line, value, "Content-Disposition",
-              //ZIP_READFIELD_STRING(line, value, "X-Addr", ..);            // Original address
+                                   r->cdispo); // Content-disposition
-              //ZIP_READFIELD_STRING(line, value, "X-Fil", ..);            // Original URI filename
               ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_);      // Original save filename
               if (line[0] != '\0') {
                 int len = r->headers ? ((int) strlen(r->headers)) : 0;
@@ -1090,8 +1088,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                 strcat(r->headers, "\r\n");
               }
             }
-          } while(offset < readSizeHeader && !lineEof);
+          } while (offset < readSizeHeader && !lineEof);
-          //totalHeader = offset;
 
           /* Previous entry */
           if (previous_save_[0] != '\0') {
@@ -1202,7 +1199,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                       strcpy(r->msg, "Cache Read Error : Read Data");
                     } else
                       *(r->adr + r->size) = '\0';
-                    //printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode);
                   } else {      // erreur
                     r->statuscode = STATUSCODE_INVALID;
                     strcpy(r->msg, "Cache Memory Error");
@@ -1484,14 +1480,12 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
               a += cache_brstr(a, firstline);
               strcpy(cache->lastmodified, firstline);
             } else {
-              // fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version);
               fclose(cache->dat);
               cache->dat = NULL;
               free(use);
               use = NULL;
             }
-          } else {              // non supporté
+          } else { // non supporté
-            // fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline);
             fclose(cache->dat);
             cache->dat = NULL;
             free(use);
@@ -1500,7 +1494,6 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
           /* */
         } else {                // Vieille version du cache
           /* */
-          // hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format");
           cache->version = 0;   // cache 1.0
           strcpy(cache->lastmodified, firstline);
         }

tests/01_engine-parse.test

@@ -106,8 +106,6 @@ grep -Eq 'srcset="j\.gif 2x"' "$saved" ||
 # inline style attribute, with the URL unquoted, double-quoted and single-quoted
 # (the quote style is preserved on rewrite). No-detect attributes (title, alt,
 # ...) are left untouched. Asserted by rewrite (deterministic), not download.
-# data-* (#201/#203) is omitted: its detection is currently nondeterministic and
-# can't be locked yet.
 site2="$tmp/attrs"
 mkdir -p "$site2"
 for f in xl ibg ibgs cex cexd cexs tt; do gif "$site2/$f.gif"; done
@@ -352,4 +350,64 @@ found "v.webm" "$out10"
 found "subs.vtt" "$out10"
 notfound "plain.gif" "$out10"
 
+# Unknown-attr (data-*) URLs (#201/#203): the script automaton state must reset at
+# </script>, or detection dies for the rest of the page after the first script.
+site11="$tmp/dataattr"
+mkdir -p "$site11"
+for f in pre post mid spdata handler handler2 jsdecoy jsdecoy2 nodecoy \
+    spalt glalt spxml jtail1 jtail2 textdecoy tagdecoy phantom; do gif "$site11/$f.gif"; done
+cat >"$site11/index.html" <<EOF
+<html><body>
+<img data-pre="file://$site11/pre.gif">
+<script>var x = 1;</script>
+<img data-post="file://$site11/post.gif">
+<img data-mid="file://$site11/mid.gif" alt="mid-tag attr, no delimiter after">
+<img data-sp = "file://$site11/spdata.gif" q=1>
+<img alt="nodecoy.gif" src="pre.gif">
+<img alt = "spalt.gif" id=x>
+<img src="pre.gif"alt="glalt.gif" id=x>
+<p xmlns:bar = "spxml.gif" id=x></p>
+<div data-json='["jtail1.gif","jtail2.gif"]' q=1></div>
+<p>t = "textdecoy.gif" q</p>
+<img ="tagdecoy.gif" q>
+<a onclick='q = 1; "h1'>x</a>
+<img data-h1="file://$site11/handler.gif">
+<a onclick='w = 1; "h2>x</a>
+<img data-h2="file://$site11/handler2.gif">
+<script>var s; s = 1; "jsdecoy.gif";</script>
+<script>var s2 = "jsdecoy2.gif" x;</script>
+<script>var f = "</script>
+<img alt="y" "phantom.gif">
+</body></html>
+EOF
+out11="$tmp/dataattr-out"
+crawl "$site11/index.html" "$out11"
+saved11=$(savedhtml "$out11")
+test -n "$saved11" || ! echo "FAIL: saved dataattr page not found" || exit 1
+grep -Fq 'data-pre="pre.gif"' "$saved11" ||
+    ! echo "FAIL #201: data-* URL before any script not detected/rewritten" || exit 1
+grep -Fq 'data-post="post.gif"' "$saved11" ||
+    ! echo "FAIL #201: data-* URL after a script not detected (state leak)" || exit 1
+grep -Fq 'data-mid="mid.gif"' "$saved11" ||
+    ! echo "FAIL #201: mid-tag data-* URL not detected" || exit 1
+grep -Fq 'data-sp = "spdata.gif"' "$saved11" ||
+    ! echo "FAIL #201: spaced-= data-* URL not detected" || exit 1
+found "handler.gif" "$out11"  # automaton reset at the handler-terminator exit
+found "handler2.gif" "$out11" # ... and at the '>' exit of an unterminated handler
+# a JS string not preceded by =/(/, is still ignored after the reset
+notfound "jsdecoy.gif" "$out11"
+# in-script, the strict follower gate still applies (intag is 1 in script bodies)
+notfound "jsdecoy2.gif" "$out11"
+# no-detect attrs stay exempt, incl. spaced-'=' and glued-attr forms
+notfound "nodecoy.gif" "$out11"
+notfound "spalt.gif" "$out11"
+notfound "glalt.gif" "$out11"
+notfound "spxml.gif" "$out11"
+# not '='-preceded: comma-list tails and out-of-tag text tokens stay ignored
+notfound "jtail2.gif" "$out11"
+notfound "textdecoy.gif" "$out11"
+notfound "tagdecoy.gif" "$out11" # a '=' glued to the tag name is not an attr
+# a </script> inside a JS string must not cause phantom fetches later
+notfound "phantom.gif" "$out11"
+
 exit 0

tests/01_engine-reconcile.test

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Cache generation reconcile policies (httrack -#test=reconcile <dir>):
+# promote a stranded old generation, keep the larger one after an aborted
+# run, and restore the old one when an update transferred nothing.
+
+set -eu
+
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+out=$(httrack -#test=reconcile "$dir")
+
+test "$out" = "cache-reconcile: OK" || {
+    echo "expected 'cache-reconcile: OK', got: $out" >&2
+    exit 1
+}

tests/01_zlib-cache-corrupt.test

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Read-side cache corruption (httrack -#test=cache-corrupt <dir>): zip byte
+# surgery (bad/oversized X-Size, blanked X-In-Cache, smashed header, garbled
+# deflate) must each be rejected per-entry, never crash, never taint the sibling.
+
+set -eu
+
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+# the smashed-header case logs expected "Corrupted cache entry" warnings on
+# stdout; the verdict is the last line
+out=$(httrack -#test=cache-corrupt "$dir" 2>/dev/null | tail -n1)
+
+test "$out" = "cache-corrupt: OK" || {
+    echo "expected 'cache-corrupt: OK', got: $out" >&2
+    exit 1
+}

tests/01_zlib-cache-writefail.test

@@ -4,10 +4,9 @@
 # POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
 # tool flags despite the #!/bin/bash above.
 
-# Cache write-failure handling (httrack -#test=cache-writefail <dir>). #174/#219.
+# Cache write-failure policy (-#test=cache-writefail <dir>). #174/#219: disk
-# A failing new.zip write (disk full) used to crash the process via assertf; it
+# full or a failure streak aborts cleanly; an isolated failure or an oversized
-# must instead stop the mirror with a fatal error (exit_xh=-1), no crash. The
+# entry is only dropped.
-# self-test asserts that; reverting the fix makes -#test=cache-writefail abort (SIGABRT) and fail.
 
 set -eu
 
@@ -22,3 +21,9 @@ printf '%s\n' "$out" | grep -qx "cache-writefail: OK" || {
     echo "expected 'cache-writefail: OK', got: $out" >&2
     exit 1
 }
+
+# A skipped entry must be warned about with its URL.
+printf '%s\n' "$out" | grep -q "entry not cached: example.com/" || {
+    echo "expected a URL-bearing skip warning" >&2
+    exit 1
+}

tests/22_local-broken-size.test

@@ -1,17 +1,19 @@
 #!/bin/bash
-# Issues #32/#41: a Content-Length that disagrees with the body warns "bogus
+# Issues #32/#41: a Content-Length that disagrees with the body warns
-# state (broken size)" and skips the cache; -%B (tolerant) accepts it.
+# "incomplete transfer" and skips the cache; -%B (tolerant) accepts it.
+
+set -euo pipefail
 
 : "${top_srcdir:=..}"
 
 # Default: warn, but the file is still written.
 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
     --found 'size/oversize.bin' \
-    --log-found 'bogus state \(broken size' \
+    --log-found 'incomplete transfer \(expected' \
     httrack 'BASEURL/size/index.html'
 
 # -%B (tolerant): no warning, file written.
 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
     --found 'size/oversize.bin' \
-    --log-not-found 'bogus state' \
+    --log-not-found 'incomplete transfer|not cached' \
     httrack 'BASEURL/size/index.html' '-%B'

tests/33_local-delayed.test

@@ -2,7 +2,7 @@
 #
 # Degenerate delayed-type paths (#5/#107 family): redirects that never resolve
 # a name must drop cleanly -- no .delayed leftovers (audited by local-crawl.sh),
-# no "bogus state" cache warnings, resolvable links still land correctly.
+# no "not cached" warnings, resolvable links still land correctly.
 
 set -euo pipefail
 
@@ -16,5 +16,5 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun --errors 0 \
     --not-found 'delayed/noloc.html' \
     --not-found 'delayed/selfloop.html' \
     --not-found 'delayed/chain9.pdf' \
-    --log-not-found 'bogus state' \
+    --log-not-found 'not cached' \
     httrack 'BASEURL/delayed/index.html'

tests/34_local-maxtime.test

@@ -7,10 +7,8 @@ set -euo pipefail
 
 : "${top_srcdir:=..}"
 
-# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
 start=$(date +%s)
 bash "$top_srcdir/tests/local-crawl.sh" \
-    --skip-delayed-audit \
     --log-found 'More than 2 seconds passed' \
     httrack 'BASEURL/trickle/index.html' -E2 -c4
 wall=$(($(date +%s) - start))

tests/35_local-maxsize.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# -M byte cap (#77): the crawl must stop with the "giving up" error and keep
+# the mirror well under the 8 x 640KB the fixture totals uncapped.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# cap = -M + the 4 in-flight files the smooth stop lets finish + one of margin
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --found bigfiles/p0.bin \
+    --max-mirror-bytes 3700000 \
+    httrack 'BASEURL/bigfiles/index.html' -M400000 -c4

tests/36_local-bigcrawl.test

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
+# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
+# family + 6 singles; the 4 planted errors write -o1 pages, not counted.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --rerun \
+    --errors 4 --files 360 \
+    --found 'big/p/95.html' \
+    --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
+    --found 'big/a/f2-2x.png' \
+    --found 'big/a/subs.vtt' \
+    --found 'big/a/font.woff2' \
+    --found 'big/a/js-data.bin' \
+    --found 'big/d/01.pdf' \
+    --found 'big/d/named.pdf' \
+    --found 'big/a/doc.pdf' \
+    --found "big/f9/caf$(printf '\xc3\xa9').html" \
+    --found 'big/f7/fa.html' \
+    --found 'big/a/ref.png' \
+    --found 'big/f6/sub/leaf.html' \
+    --found 'big/f1/dir/index.html' \
+    --found 'big/f10/empty.html' \
+    --found 'big/indexd41d.html' \
+    --found 'big/a/i0a.png' \
+    --not-found 'big/x/og' \
+    --not-found 'big/x/tw' \
+    --not-found 'big/x/jsonld.png' \
+    --not-found 'big/x/never-scanned.png' \
+    --not-found 'big/x/atom-only.html' \
+    --not-found 'big/x/sitemap-only.html' \
+    --not-found 'big/x/form-target.html' \
+    --not-found 'big/x/formact' \
+    --not-found 'big/x/ping' \
+    --not-found 'big/x/aj.jar' \
+    --not-found 'big/x/bj.jar' \
+    --not-found 'big/x/is1.png' \
+    --not-found 'big/x/concat.html' \
+    --file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
+    --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
+    --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \
+    --file-not-matches 'big/p/1.html' 'href="/big/' \
+    --log-not-found 'not cached|[Pp]anic|assert' \
+    --log-found '\(404\) at link [^ ]*/big/e/404\.html' \
+    --log-found '\(410\) at link [^ ]*/big/e/410\.html' \
+    --log-found '\(500\) at link [^ ]*/big/e/500\.html' \
+    --log-found 'decompressing.*big/e/gztrunc\.html' \
+    --log-found ', no files updated' \
+    --max-mirror-bytes 700000 \
+    --min-mirror-bytes 500000 \
+    httrack 'BASEURL/big/index.html' --retries=0 -c8 -%c100 -A100000000

tests/37_local-cache-outage.test

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# An update run against a dead server must not destroy the cache: the no-data
+# rollback restores the previous hts-cache generation (zip caches lost it).
+
+set -eu
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun-dead \
+    --found 'simple/basic.html' \
+    httrack 'BASEURL/simple/basic.html'

tests/38_local-update-304.test

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# An all-304 update of a tiny site (headers under the 32K rollback threshold)
+# is a healthy run: it must not trip the no-data rollback as a fake outage.
+
+set -eu
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
+    --log-found 'no files updated' \
+    --log-not-found 'No data seems to have been transferred' \
+    --found 'mini304/index.html' --found 'mini304/page.html' \
+    httrack 'BASEURL/mini304/index.html'

tests/39_local-delayed-cancel.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Cancelled delayed-type-checks must not orphan .delayed placeholders (#483).
+# Timing-dependent (hence two tries); -A keeps the window reachable.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+for _ in 1 2; do
+    bash "$top_srcdir/tests/local-crawl.sh" \
+        httrack 'BASEURL/dcancel/index.html' -E1 -c4 -A25000
+done

tests/Makefile.am

@@ -48,6 +48,7 @@ TESTS = \
 	01_engine-parse.test \
 	01_engine-pause.test \
 	01_engine-rcfile.test \
+	01_engine-reconcile.test \
 	01_engine-redirect.test \
 	01_engine-relative.test \
 	01_engine-robots.test \
@@ -63,6 +64,7 @@ TESTS = \
 	01_engine-useragent.test \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
+	01_zlib-cache-corrupt.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
 	01_zlib-savename-cached.test \
@@ -97,6 +99,11 @@ TESTS = \
 	31_local-javaclass.test \
 	32_local-cdispo.test \
 	33_local-delayed.test \
-	34_local-maxtime.test
+	34_local-maxtime.test \
+	35_local-maxsize.test \
+	36_local-bigcrawl.test \
+	37_local-cache-outage.test \
+	38_local-update-304.test \
+	39_local-delayed-cancel.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -16,13 +16,17 @@
 #       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
+#       --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
+# --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
+# --rerun-dead re-runs with the server stopped: the no-data rollback must
+# restore the previous hts-cache generation byte-identical.
 
 set -u
 
@@ -35,6 +39,7 @@ key="${testdir}/server.key"
 tls=
 verbose=
 rerun=
+rerun_dead=
 tmpdir=
 serverpid=
 crawlpid=
@@ -92,7 +97,6 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
 # --- parse leading control flags --------------------------------------------
 declare -a audit=()
 declare -a cookies=()
-skip_delayed_audit=""
 scheme=http
 pos=0
 args=("$@")
@@ -100,7 +104,8 @@ nargs=$#
 while test "$pos" -lt "$nargs"; do
     case "${args[$pos]}" in
     --debug) verbose=1 ;;
-    --rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
+    --rerun) rerun=1 ;;           # run httrack a second time (update pass) before auditing
+    --rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
     --no-purge)
         nopurge=1
         audit+=("--no-purge")
@@ -117,14 +122,11 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
-    --skip-delayed-audit)
-        skip_delayed_audit=1
-        ;;
     --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --found | --not-found | --directory | --log-found | --log-not-found)
+    --found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes | --min-mirror-bytes)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -239,6 +241,43 @@ if test -n "$rerun"; then
     fi
 fi
 
+# --- optional dead pass: server stopped, the cache must survive the rollback --
+if test -n "$rerun_dead"; then
+    zip="${out}/hts-cache/new.zip"
+    test -s "$zip" || die "no cache was written by the first pass"
+    cp "$zip" "${tmpdir}/cache-before.zip"
+    cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
+    kill "$serverpid" 2>/dev/null
+    wait "$serverpid" 2>/dev/null
+    serverpid=
+    info "re-running httrack against the stopped server"
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
+        "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
+    crawlpid=$!
+    wait "$crawlpid" || true
+    crawlpid=
+    result "OK (dead pass ran)"
+    # The dead pass must have gone through the no-data rollback, not bailed out
+    # before the mirror loop (which would leave the cache trivially untouched).
+    info "checking the dead pass hit the rollback"
+    if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
+        result "OK"
+    else
+        result "rollback notice not found in hts-log.txt"
+        exit 1
+    fi
+    info "checking the previous cache generation was restored"
+    if cmp -s "$zip" "${tmpdir}/cache-before.zip" &&
+        test ! -e "${out}/hts-cache/old.zip"; then
+        result "OK"
+    else
+        result "new.zip differs from the pre-outage cache (or old.zip left behind)"
+        exit 1
+    fi
+    # Audits below describe the healthy crawl, not the dead pass.
+    cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
+fi
+
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
 hostroot=
 for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
@@ -250,15 +289,12 @@ done
 test -n "$hostroot" || die "could not find host root under $out"
 debug "host root: $hostroot"
 
-# A completed crawl must leave no .delayed temporaries (issue #107).
+# No crawl, even a cancelled one, may leave .delayed temporaries (#107, #483).
-# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
+info "checking for leftover .delayed files"
-if test -z "$skip_delayed_audit"; then
+leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
-    info "checking for leftover .delayed files"
+if test -z "$leftovers"; then result "OK"; else
-    leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
+    result "leftover: $leftovers"
-    if test -z "$leftovers"; then result "OK"; else
+    exit 1
-        result "leftover: $leftovers"
-        exit 1
-    fi
 fi
 
 # --- audit -------------------------------------------------------------------
@@ -316,6 +352,24 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
+    --max-mirror-bytes)
+        i=$((i + 1))
+        sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
+        info "checking mirror size ${sz} <= ${audit[$i]} bytes"
+        if test "$sz" -le "${audit[$i]}"; then result "OK"; else
+            result "mirror too big"
+            exit 1
+        fi
+        ;;
+    --min-mirror-bytes)
+        i=$((i + 1))
+        sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
+        info "checking mirror size ${sz} >= ${audit[$i]} bytes"
+        if test "$sz" -ge "${audit[$i]}"; then result "OK"; else
+            result "mirror too small"
+            exit 1
+        fi
+        ;;
     --file-matches)
         path="${audit[$((i + 1))]}"
         i=$((i + 2))

tests/local-server.py

@@ -15,6 +15,7 @@ stdlib only (http.server + ssl) -- no new build or runtime dependency.
 
 import argparse
 import gzip
+import hashlib
 import os
 import time
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
@@ -42,6 +43,416 @@ PAGE = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
 """
 
 
+# --- /big/ seeded pseudo-site (36_local-bigcrawl) ---------------------------
+# Deterministic ~360-file tree; bodies derive from sha256(BIG_SEED, name) so
+# every run serves identical content and the test pins exact counts.
+BIG_SEED = "bigcrawl-lite-1"
+BIG_PAGES = 96
+BIG_FANOUT = 4
+# Fixed validator: a matching If-Modified-Since gets 304, so the update pass
+# revalidates instead of re-downloading.
+BIG_LASTMOD = "Mon, 01 Jan 2024 00:00:00 GMT"
+
+BIG_CTYPES = {
+    "html": "text/html",
+    "css": "text/css",
+    "js": "application/x-javascript",
+    "png": "image/png",
+    "gif": "image/gif",
+    "jpg": "image/jpeg",
+    "webp": "image/webp",
+    "pdf": "application/pdf",
+    "woff2": "font/woff2",
+    "mp4": "video/mp4",
+    "webm": "video/webm",
+    "mp3": "audio/mpeg",
+    "vtt": "text/vtt",
+    "xml": "text/xml",
+    "svg": "image/svg+xml",
+    "jar": "application/java-archive",
+    "bin": "application/octet-stream",
+}
+
+# Honest magic bytes per claimed type so the #478 sniff never contests.
+BIG_MAGIC = {
+    "png": b"\x89PNG\r\n\x1a\n",
+    "gif": b"GIF89a",
+    "jpg": b"\xff\xd8\xff\xe0",
+    "webp": b"RIFF\x10\x27\x00\x00WEBPVP8 ",
+    "pdf": b"%PDF-1.4\n",
+    "woff2": b"wOF2",
+    "mp4": b"\x00\x00\x00\x18ftypmp42",
+    "webm": b"\x1a\x45\xdf\xa3",
+    "mp3": b"ID3\x04\x00\x00\x00\x00\x00\x00",
+    "jar": b"PK\x03\x04",
+}
+
+
+def big_blob(name, size):
+    out = b""
+    n = 0
+    while len(out) < size:
+        out += hashlib.sha256(f"{BIG_SEED}/{name}/{n}".encode()).digest()
+        n += 1
+    return out[:size]
+
+
+def big_asset(name):
+    ext = name.rsplit(".", 1)[-1]
+    size = 200 + int(hashlib.sha256(name.encode()).hexdigest(), 16) % 3800
+    raw = big_blob(name, size)
+    if ext in ("css", "js", "txt"):
+        return b"/* " + raw.hex().encode() + b" */"
+    return BIG_MAGIC.get(ext, b"") + raw
+
+
+def big_html(title, inner):
+    page = (
+        "<!DOCTYPE html><html><head><title>%s</title></head><body>\n%s\n</body></html>"
+        % (
+            title,
+            inner,
+        )
+    )
+    return page.encode()
+
+
+def _hexfill(name):
+    return big_blob(name, 160).hex()
+
+
+HOME = '<a href="/big/index.html">home</a>'
+
+BIG_TEXT_ASSETS = {
+    "site.css": (
+        "body { background: url(bg.png); } /* %s */" % _hexfill("site.css"),
+        "text/css",
+    ),
+    "print.css": ("p { margin: 0; } /* %s */" % _hexfill("print.css"), "text/css"),
+    "blk.css": (
+        '@import "blk2.css";\n'
+        '@font-face { font-family: big; src: local("Nope Sans"), '
+        'url(font.woff2) format("woff2"); }\n'
+        "/* %s */" % _hexfill("blk.css"),
+        "text/css",
+    ),
+    # Absolute url() must come back relative after the rewrite (test greps it);
+    # the \/ escapes collapse to an already-linked URL if taken literally.
+    "blk2.css": (
+        "body { background: url(/big/a/blk2-bg.png); }\n"
+        "i { background: url(/big\\/a\\/bg.png); }\n"
+        "/* %s */" % _hexfill("blk2.css"),
+        "text/css",
+    ),
+    # .open() grabs its first arg only (a method there is rejected, #218), so
+    # the window.open single-URL form is the token-detected shape.
+    "app.js": (
+        'var im = new Image(); im.src = "/big/a/js-img.png";\n'
+        'function pop() { window.open("/big/a/js-data.bin"); }\n'
+        "// %s\n" % _hexfill("app.js"),
+        "application/x-javascript",
+    ),
+    "heavy.js": (
+        'var h = new Image(); h.src = "/big/a/js1.png";\n'
+        'function nav() { location.href = "/big/p/1.html"; }\n'
+        'function pop() { window.open("/big/a/js2.bin"); }\n'
+        "// %s\n" % _hexfill("heavy.js"),
+        "application/x-javascript",
+    ),
+    # text/javascript is fetched but never scanned: the URL inside must stay
+    # out of the mirror.
+    "decoy.js": (
+        'var d = new Image(); d.src = "/big/x/never-scanned.png";\n',
+        "text/javascript",
+    ),
+    "subs.vtt": ("WEBVTT\n\n00:00.000 --> 00:01.000\nbig\n", "text/vtt"),
+    "logo.svg": (
+        '<svg xmlns="http://www.w3.org/2000/svg" width="4" height="4">'
+        '<image href="ref.png" width="4" height="4"/></svg>',
+        "image/svg+xml",
+    ),
+}
+
+
+def _fam_feeds(port):
+    return (
+        '<link rel="alternate" type="application/rss+xml" href="/big/f12/rss.xml">'
+        '<a href="/big/f12/atom.xml">atom</a>'
+        '<a href="/big/f12/sitemap.xml">sitemap</a>'
+    )
+
+
+def _fam_plain(port):
+    return (
+        '<a href="../f1/one.html">one</a>'
+        '<a href="./two.html">two</a>'
+        '<a href="../../big/f1/tri.html">tri</a>'
+        '<a href="/big/f1/abs.html">abs</a>'
+        '<a href="/big/f1/list.html">list</a>'
+        '<a href="/big/f1/list.html?page=2">p2</a>'
+        '<a href="/big/f1/list.html?page=3&amp;sort=asc">p3</a>'
+        '<a href="/big/f1/dir">dir</a>'
+        '<a href="">self</a><a href="#">frag</a>'
+        '<a href="mailto:big@example.com">mail</a>'
+        '<a href="tel:+15551234">tel</a>'
+        '<a href="data:text/plain;base64,aGk=">data</a>'
+    )
+
+
+def _fam_srcset(port):
+    return (
+        '<img src="/big/a/f2-base.png">'
+        '<img srcset="/big/a/f2-1x.png 1x, /big/a/f2-2x.png 2x"'
+        ' src="/big/a/f2-base.png">'
+        '<img data-srcset="/big/a/f2-1x.png 1x, /big/a/f2-2x.png 2x"'
+        ' src="/big/a/f2-base.png" loading="lazy">'
+        '<picture><source type="image/webp" srcset="/big/a/f2-alt.webp">'
+        '<img src="/big/a/f2-base.png"></picture>'
+    )
+
+
+def _fam_media(port):
+    return (
+        '<video src="/big/a/clip.mp4" poster="/big/a/poster.jpg">'
+        '<source src="/big/a/clip.webm" type="video/webm">'
+        '<track src="/big/a/subs.vtt" kind="subtitles" srclang="en">'
+        "</video>"
+        '<audio><source src="/big/a/tune.mp3" type="audio/mpeg"></audio>'
+    )
+
+
+def _fam_css(port):
+    # image-set with descriptors is a proven-safe decoy (engine-surface §6).
+    return (
+        '<link rel="stylesheet" href="/big/a/print.css" media="print">'
+        '<div style="background:url(/big/a/attr-bg.png)">styled</div>'
+        '<style>@import "/big/a/blk.css"; h1 { background: url(/big/a/blk-bg.gif); }'
+        ' h2 { background-image: image-set("/big/x/is1.png" 1x, "/big/x/is2.png" 2x); }'
+        "</style>"
+    )
+
+
+def _fam_js(port):
+    # The concatenated string is rejected by the scanner (no single literal).
+    return (
+        '<script src="/big/a/heavy.js"></script>'
+        '<script src="/big/a/decoy.js"></script>'
+        "<script>document.write('<a href=\"/big/f5/dw.html\">dw</a>');\n"
+        'var nope = "xx-" + "/big/x/concat.html";</script>'
+    )
+
+
+def _fam_meta(port):
+    # Extensionless decoy targets stay unfetchable even if the aggressive
+    # parser fires (no known extension, no scheme: rejected in every state).
+    return (
+        '<meta http-equiv="refresh" content="2;URL=/big/f6/refreshed.html">'
+        '<a href="/big/f6/based.html">based</a>'
+        '<meta property="og:image" content="/big/x/og">'
+        '<meta name="twitter:image" content="/big/x/tw">'
+        '<script type="application/ld+json">'
+        '{"@type": "Thing", "image": "/big/x/jsonld.png"}</script>'
+    )
+
+
+def _fam_legacy(port):
+    # Comma-valued applet archive is rejected whole by the engine (decoy).
+    return (
+        '<a href="/big/f7/frames.html">frames</a>'
+        '<img src="/big/a/map.gif" usemap="#m">'
+        '<map name="m">'
+        '<area shape="rect" coords="0,0,9,9" href="/big/f7/area.html"></map>'
+        '<embed src="/big/a/e.pdf" type="application/pdf" width="9" height="9">'
+        '<object data="/big/a/o.pdf" type="application/pdf"></object>'
+        '<applet archive="/big/x/aj.jar,/big/x/bj.jar" width="1" height="1"></applet>'
+    )
+
+
+def _fam_svg(port):
+    return (
+        '<svg width="9" height="9">'
+        '<image href="/big/a/svg-in.png" width="4" height="4"/>'
+        '<use xlink:href="#icon"/></svg>'
+        '<img src="/big/a/logo.svg">'
+    )
+
+
+def _fam_i18n(port):
+    return (
+        '<a href="/big/f9/caf%C3%A9.html">cafe</a>'
+        '<a href="/big/f9/latin1.html">latin1</a>'
+        '<a href="/big/f9/metaonly.html">meta</a>'
+        '<a href="/big/f9/bom.html">bom</a>'
+    )
+
+
+def _fam_http(port):
+    return (
+        '<a href="/big/r/hop1">chain</a>'
+        '<a href="/big/r/get42">get42</a>'
+        '<a href="/big/d/01">d01</a>'
+        '<a href="/big/d/02">d02</a>'
+        '<a href="/big/f10/empty.html">empty</a>'
+        '<a href="/big/d/dl">dl</a>'
+    )
+
+
+def _fam_forms(port):
+    # GET form action is rewritten but never fetched; formaction/ping are
+    # outside the attribute tables (decoys).
+    return (
+        '<form action="/big/x/form-target.html" method="get">'
+        '<input type="text" name="q">'
+        '<input type="image" src="/big/a/btn.png" alt="go"></form>'
+        '<a href="/big/f11/page.html">bare</a>'
+        '<a href="/big/f11/page.html?utm_source=news&amp;utm_medium=mail">utm</a>'
+        '<a href="/big/f11/sess.html?PHPSESSID=deadbeef123">sess</a>'
+        '<button formaction="/big/x/formact">go</button>'
+        '<a href="/big/f11/page.html" ping="/big/x/ping">ping</a>'
+    )
+
+
+BIG_FAMILIES = [
+    _fam_feeds,
+    _fam_plain,
+    _fam_srcset,
+    _fam_media,
+    _fam_css,
+    _fam_js,
+    _fam_meta,
+    _fam_legacy,
+    _fam_svg,
+    _fam_i18n,
+    _fam_http,
+    _fam_forms,
+]
+
+
+def big_link(m, style):
+    return ["%d.html" % m, "../p/%d.html" % m, "/big/p/%d.html" % m][style]
+
+
+def big_page(n, port):
+    style = n % 3
+    home = ["../index.html", "/big/index.html", "../index.html"][style]
+    parts = ['<a href="%s">home</a>' % home]
+    if n > 0:
+        parts.append('<a href="%s">up</a>' % big_link((n - 1) // BIG_FANOUT, style))
+    for c in range(n * BIG_FANOUT + 1, n * BIG_FANOUT + BIG_FANOUT + 1):
+        if c < BIG_PAGES:
+            parts.append('<a href="%s">p%d</a>' % (big_link(c, style), c))
+    parts.append('<link rel="stylesheet" href="/big/a/site.css">')
+    parts.append('<script src="/big/a/app.js"></script>')
+    exts = ["png", "gif", "jpg"]
+    ia = "/big/a/i%da.%s" % (n, exts[n % 3])
+    ib = "/big/a/i%db.%s" % (n, exts[(n + 1) % 3])
+    # Rotate the second-image construct across deterministic table attributes.
+    con = n % 4
+    if con == 0:
+        parts.append('<img src="%s"><img src="%s">' % (ia, ib))
+    elif con == 1:
+        parts.append(
+            '<img src="%s"><table background="%s"><tr><td>t</td></tr></table>'
+            % (ia, ib)
+        )
+    elif con == 2:
+        parts.append('<img src="%s"><img src="%s" data-src="%s">' % (ia, ia, ib))
+    else:
+        parts.append(
+            '<img src="%s" loading="lazy"><video poster="%s"></video>' % (ia, ib)
+        )
+    parts.append(BIG_FAMILIES[n % 12](port))
+    return big_html("p%d" % n, "\n".join(parts))
+
+
+def big_index(port):
+    return big_html(
+        "big index",
+        '<link rel="stylesheet" href="/big/a/site.css">'
+        '<script src="/big/a/app.js"></script>'
+        '<a href="p/0.html">root</a>'
+        '<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
+        '<a href="/big/f1/long.html?x=%s">long</a>'
+        '<a href="/big/f1/gzok.html">gzok</a>'
+        '<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
+        '<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
+        '<a href="/big/e/404.html">e404</a>'
+        '<a href="/big/e/410.html">e410</a>'
+        '<a href="/big/e/500.html">e500</a>'
+        '<a href="/big/e/gztrunc.html">gzt</a>'
+        '<a href="?">query</a>' % ("a" * 900, port, port),
+    )
+
+
+BIG_REDIRECTS = {
+    "/big/r/hop1": (301, "/big/r/hop2"),
+    "/big/r/hop2": (302, "/big/f10/land.html"),
+    "/big/r/get42": (301, "/big/a/doc.pdf"),
+    "/big/f1/dir": (301, "/big/f1/dir/"),
+}
+
+BIG_SIMPLE_PAGES = {
+    "/big/p/two.html": "dot-slash target",
+    "/big/f1/one.html": "one",
+    "/big/f1/tri.html": "tri",
+    "/big/f1/abs.html": "abs",
+    "/big/f1/dir/": "dir index",
+    "/big/f1/long.html": "long",
+    "/big/f1/gzok.html": "gzok",
+    "/big/f1/protorel.html": "protorel",
+    "/big/f1/abshost.html": "abshost",
+    "/big/f5/dw.html": "dw target",
+    "/big/f6/refreshed.html": "refreshed",
+    "/big/f6/sub/leaf.html": "leaf",
+    "/big/f7/fa.html": "frame a",
+    "/big/f7/fb.html": "frame b",
+    "/big/f7/fn.html": "noframes",
+    "/big/f7/area.html": "area",
+    "/big/f10/land.html": "landed",
+    "/big/f11/page.html": "the page",
+    "/big/f11/sess.html": "the sess page",
+}
+
+# Extensionless downloads: name resolution is wire-type driven (#478 contract).
+BIG_DOWNLOADS = {
+    "/big/d/01": ("pdf", None),
+    "/big/d/02": ("png", None),
+    "/big/d/dl": ("pdf", 'attachment; filename="named.pdf"'),
+}
+
+
+def _big_rss(port):
+    # purl.org marker makes the feed parse; item URLs are already-linked pages.
+    return (
+        '<?xml version="1.0"?>\n'
+        '<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">\n'
+        "<channel><title>big</title><link>http://127.0.0.1:%d/big/index.html</link>\n"
+        "<item><title>i1</title><link>http://127.0.0.1:%d/big/p/1.html</link>\n"
+        '<enclosure url="http://127.0.0.1:%d/big/p/2.html" type="text/html"/></item>\n'
+        "</channel></rss>\n" % (port, port, port)
+    ).encode()
+
+
+def _big_atom(port):
+    # No purl marker: emitted verbatim, its URL must never be fetched.
+    return (
+        '<?xml version="1.0"?>\n'
+        '<feed xmlns="http://www.w3.org/2005/Atom"><title>big</title>\n'
+        "<entry><title>e1</title>"
+        '<link href="http://127.0.0.1:%d/big/x/atom-only.html"/>'
+        "</entry></feed>\n" % port
+    ).encode()
+
+
+def _big_sitemap(port):
+    return (
+        '<?xml version="1.0"?>\n'
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
+        "<url><loc>http://127.0.0.1:%d/big/x/sitemap-only.html</loc></url>\n"
+        "</urlset>\n" % port
+    ).encode()
+
+
 class Handler(SimpleHTTPRequestHandler):
     # Quieter logging; the launcher captures httrack's own log anyway.
     def log_message(self, fmt, *args):
@@ -370,7 +781,7 @@ class Handler(SimpleHTTPRequestHandler):
         self.send_raw(b"", "text/html")
 
     # broken Content-Length (#32/#41): declared size != bytes sent. httrack
-    # warns "bogus state (broken size)" and skips the cache unless -%B.
+    # warns "incomplete transfer" and skips the cache unless -%B.
     def route_size_index(self):
         self.send_html('\t<a href="oversize.bin">over</a>\n')
 
@@ -420,6 +831,16 @@ class Handler(SimpleHTTPRequestHandler):
     def route_redir_target(self):
         self.send_raw(b"<html><body>redirect target</body></html>\n", "text/html")
 
+    # --- /mini304/: tiny fully-cacheable site (an update gets only 304s) ---
+    def route_mini304_index(self):
+        self.big_send(
+            b'<html><body>\n\t<a href="page.html">page</a>\n</body></html>\n',
+            "text/html",
+        )
+
+    def route_mini304_page(self):
+        self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
+
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -468,11 +889,15 @@ class Handler(SimpleHTTPRequestHandler):
     # so only an engine-side abort can end the crawl.
     TRICKLE_SECONDS = 60
 
-    def route_trickle_index(self):
+    def send_bin_index(self):
+        """Index page linking p0.bin..p7.bin (shared by trickle and bigfiles)."""
         self.send_html(
             "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
         )
 
+    def route_trickle_index(self):
+        self.send_bin_index()
+
     def route_trickle_page(self):
         self.send_response(200)
         self.send_header("Content-Type", "application/octet-stream")
@@ -488,6 +913,35 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
+    # #483: trickled .bin pages so the -E stop lands in the type waiter's
+    # unlock-to-patch window with body bytes pending.
+    def route_dcancel_index(self):
+        self.send_bin_index()
+
+    def route_dcancel_page(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", "4096")
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            for _ in range(32):
+                self.wfile.write(b"z" * 128)
+                self.wfile.flush()
+                time.sleep(0.05)
+        except OSError:
+            pass
+
+    # -M byte cap (#77): large fast files so a crawl overruns -M immediately.
+    BIGFILE_BYTES = 640 * 1024
+
+    def route_bigfiles_index(self):
+        self.send_bin_index()
+
+    def route_bigfile(self):
+        self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -542,6 +996,24 @@ class Handler(SimpleHTTPRequestHandler):
         "/trickle/p5.bin": route_trickle_page,
         "/trickle/p6.bin": route_trickle_page,
         "/trickle/p7.bin": route_trickle_page,
+        "/dcancel/index.html": route_dcancel_index,
+        "/dcancel/p0.bin": route_dcancel_page,
+        "/dcancel/p1.bin": route_dcancel_page,
+        "/dcancel/p2.bin": route_dcancel_page,
+        "/dcancel/p3.bin": route_dcancel_page,
+        "/dcancel/p4.bin": route_dcancel_page,
+        "/dcancel/p5.bin": route_dcancel_page,
+        "/dcancel/p6.bin": route_dcancel_page,
+        "/dcancel/p7.bin": route_dcancel_page,
+        "/bigfiles/index.html": route_bigfiles_index,
+        "/bigfiles/p0.bin": route_bigfile,
+        "/bigfiles/p1.bin": route_bigfile,
+        "/bigfiles/p2.bin": route_bigfile,
+        "/bigfiles/p3.bin": route_bigfile,
+        "/bigfiles/p4.bin": route_bigfile,
+        "/bigfiles/p5.bin": route_bigfile,
+        "/bigfiles/p6.bin": route_bigfile,
+        "/bigfiles/p7.bin": route_bigfile,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,
@@ -560,8 +1032,150 @@ class Handler(SimpleHTTPRequestHandler):
         "/redir/index.html": route_redir_index,
         "/redir/go.php": route_redir_go,
         "/redir/target.html": route_redir_target,
+        "/mini304/index.html": route_mini304_index,
+        "/mini304/page.html": route_mini304_page,
     }
 
+    # --- /big/ seeded pseudo-site ------------------------------------------
+
+    def big_send(self, body, ctype, code=200, extra=()):
+        if code == 200 and self.headers.get("If-Modified-Since") == BIG_LASTMOD:
+            self.send_response(304)
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+            return
+        self.send_response(code)
+        if code == 200:
+            self.send_header("Last-Modified", BIG_LASTMOD)
+        self.send_header("Content-Type", ctype)
+        self.send_header("Content-Length", str(len(body)))
+        for name, value in extra:
+            self.send_header(name, value)
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(body)
+
+    def big_error(self, code, reason):
+        body = big_html("error", "<p>%d</p>%s" % (code, HOME))
+        self.big_send(body, "text/html", code=code, extra=[("X-Reason", reason)])
+
+    def route_big(self):
+        split = urlsplit(self.path)
+        path = unquote(split.path)
+        port = self.server.server_address[1]
+        if path in BIG_REDIRECTS:
+            code, location = BIG_REDIRECTS[path]
+            self.send_response(code)
+            self.send_header("Location", location)
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+        elif path == "/big/index.html":
+            self.big_send(big_index(port), "text/html")
+        elif path in BIG_SIMPLE_PAGES:
+            body = big_html(path, "<p>%s</p>%s" % (BIG_SIMPLE_PAGES[path], HOME))
+            if path == "/big/f1/gzok.html":
+                self.big_send(
+                    gzip.compress(body, mtime=0),
+                    "text/html",
+                    extra=[("Content-Encoding", "gzip")],
+                )
+            else:
+                self.big_send(body, "text/html")
+        elif path == "/big/f1/list.html":
+            # Pagination: distinct content per query string.
+            body = big_html("list", "<p>listing %s</p>%s" % (split.query or "1", HOME))
+            self.big_send(body, "text/html")
+        elif path == "/big/f6/based.html":
+            self.big_send(
+                big_html(
+                    "based",
+                    '<base href="http://127.0.0.1:%d/big/f6/sub/">'
+                    '<a href="leaf.html">leaf</a>' % port,
+                ),
+                "text/html",
+            )
+        elif path == "/big/f7/frames.html":
+            self.big_send(
+                b'<html><frameset cols="50%,50%"><frame src="fa.html">'
+                b'<frame src="fb.html"><noframes><body><a href="fn.html">fn</a>'
+                b"</body></noframes></frameset></html>",
+                "text/html",
+            )
+        elif path == "/big/f9/café.html":
+            self.big_send(big_html("cafe", "<p>cafe</p>%s" % HOME), "text/html")
+        elif path == "/big/f9/latin1.html":
+            self.big_send(
+                b"<html><body><p>caf\xe9 latin</p></body></html>",
+                "text/html; charset=ISO-8859-1",
+            )
+        elif path == "/big/f9/metaonly.html":
+            self.big_send(
+                '<html><head><meta charset="utf-8"></head>'
+                "<body><p>café meta</p></body></html>".encode(),
+                "text/html",
+            )
+        elif path == "/big/f9/bom.html":
+            self.big_send(
+                b"\xef\xbb\xbf" + big_html("bom", "<p>bom</p>%s" % HOME), "text/html"
+            )
+        elif path == "/big/f10/empty.html":
+            self.big_send(b"", "text/html")
+        elif path == "/big/f12/rss.xml":
+            self.big_send(_big_rss(port), "text/xml")
+        elif path == "/big/f12/atom.xml":
+            self.big_send(_big_atom(port), "application/xml")
+        elif path == "/big/f12/sitemap.xml":
+            self.big_send(_big_sitemap(port), "text/xml")
+        elif path.startswith("/big/p/"):
+            try:
+                n = int(path[len("/big/p/") : -len(".html")])
+            except ValueError:
+                n = -1
+            if 0 <= n < BIG_PAGES and path.endswith(".html"):
+                self.big_send(big_page(n, port), "text/html")
+            else:
+                self.big_error(404, "no such page")
+        elif path.startswith("/big/a/") or path.startswith("/big/x/"):
+            name = path[len("/big/a/") :]
+            if path.startswith("/big/a/") and name in BIG_TEXT_ASSETS:
+                text, ctype = BIG_TEXT_ASSETS[name]
+                self.big_send(text.encode(), ctype)
+            elif name.endswith(".html"):
+                # Decoy targets 200 so a parser leak becomes a mirror file.
+                self.big_send(big_html(name, "<p>%s</p>" % name), "text/html")
+            else:
+                ext = name.rsplit(".", 1)[-1]
+                ctype = BIG_CTYPES.get(ext, "application/octet-stream")
+                self.big_send(big_asset(name), ctype)
+        elif path in BIG_DOWNLOADS:
+            ext, cdispo = BIG_DOWNLOADS[path]
+            extra = [("Content-Disposition", cdispo)] if cdispo else []
+            self.big_send(
+                big_asset(path[len("/big/") :] + "." + ext),
+                BIG_CTYPES[ext],
+                extra=extra,
+            )
+        elif path == "/big/e/404.html":
+            self.big_error(404, "Not Found")
+        elif path == "/big/e/410.html":
+            self.big_error(410, "Gone")
+        elif path == "/big/e/500.html":
+            self.big_error(500, "Server Error")
+        elif path == "/big/e/gztrunc.html":
+            # Half a gzip stream, honest Content-Length: decode fails, and the
+            # missing Last-Modified keeps it the one uncacheable resource.
+            full = gzip.compress(big_html("gz", "x" * 3000), mtime=0)
+            body = full[: len(full) // 2]
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Encoding", "gzip")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(body)
+        else:
+            self.big_error(404, "no such big path")
+
     # --- dispatch ----------------------------------------------------------
 
     def reject_fragment(self):
@@ -577,6 +1191,9 @@ class Handler(SimpleHTTPRequestHandler):
     def dispatch(self):
         self._set_cookies = []
         path = urlsplit(self.path).path
+        if path.startswith("/big/"):
+            self.route_big()
+            return True
         # Match percent-encoded paths (accented #157 route) by their decoded form.
         handler = self.ROUTES.get(path) or self.ROUTES.get(unquote(path))
         if handler is not None:

tools/mkdeb.sh

@@ -211,7 +211,9 @@ main() {
     # lintian ourselves below as the real gate.
     local -a debuild_opts=(--no-lintian)
     local -a build_opts=()
-    [[ $source_only -eq 1 ]] && build_opts+=(-S)
+    # -d: a source build runs no debhelper, so don't require Build-Depends
+    # locally (the buildds and the --sbuild gate enforce them).
+    [[ $source_only -eq 1 ]] && build_opts+=(-S -d)
     if [[ $unsigned -eq 1 ]]; then
         build_opts+=(-us -uc)
     else
@@ -234,12 +236,15 @@ main() {
 
     # The real lintian gate (debuild only reports, it does not fail on tags).
     # --profile debian: CI runners are Ubuntu, whose vendor data would wrongly
-    # reject the Debian "unstable" distribution. newer-standards-version only
+    # reject the Debian "unstable" distribution. Suppressed tags are stale-local-
-    # means the local lintian is older than the buildds', not a package
+    # lintian skew, not package defects: newer-standards-version, and
-    # defect, so suppress it. set -e turns any error/warning tag into a failure.
+    # recommended-field (old lintian still wants the Priority field the sid
+    # lintian in CI accepts dropping). set -e turns any error/warning tag into
+    # a failure.
     info "running lintian gate (--fail-on=error,warning)"
     lintian --profile debian -I -i --fail-on=error,warning \
-        --suppress-tags newer-standards-version "${changes[@]}"
+        --suppress-tags newer-standards-version,recommended-field \
+        "${changes[@]}"
 
     dcmd cp -- "${changes[@]}" "$outdir/"