sitemap: translate the new GUI strings into the remaining 28 locales (#738)

The feature PR added the four LANG_SITEMAP* entries to lang.def with English
and Francais only; every other locale fell back to English in the WebHTTrack
form. Each file is written in its own declared charset.

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

.github/workflows/ci.yml

@@ -46,32 +46,10 @@ jobs:
       - name: Configure
         run: |
           set -euo pipefail
-          # Regenerate: configure and the Makefile.in's are not tracked.
+          # Regenerate from configure.ac/Makefile.am to validate them; the
+          # committed generated files already let a plain checkout build.
           autoreconf -fi
-          # Disabling zlib must fail here rather than at link with a pile of
-          # undefined minizip references (#735). Both spellings, so a rewrite
-          # cannot keep one and lose the other. Probed out-of-tree to leave
-          # nothing behind.
-          nozlib="$RUNNER_TEMP/nozlib"
-          for arg in --without-zlib --with-zlib=no; do
-            rm -rf "$nozlib" && mkdir -p "$nozlib"
-            if (cd "$nozlib" && "$GITHUB_WORKSPACE/configure" "$arg" >out.log 2>&1); then
-              echo "::error::configure $arg succeeded; it must be rejected"
-              exit 1
-            fi
-            # ... and for the stated reason, not an unrelated configure failure.
-            grep -q "zlib cannot be disabled" "$nozlib/out.log" \
-              || { cat "$nozlib/out.log"; exit 1; }
-          done
           ./configure
-          # Same dead end from the compile side. The bare compile is the
-          # control: without it a broken probe would pass vacuously.
-          hdr='#include "htsglobal.h"'
-          echo "$hdr" | $CC -I. -Isrc -fsyntax-only -xc -
-          if echo "$hdr" | $CC -DHTS_USEZLIB=0 -I. -Isrc -fsyntax-only -xc - 2>/dev/null; then
-            echo "::error::-DHTS_USEZLIB=0 compiled; htsglobal.h must reject it"
-            exit 1
-          fi
           # a missing decoder would silently drop the coding from Accept-Encoding
           grep -q "define HTS_USEBROTLI 1" config.h
           grep -q "define HTS_USEZSTD 1" config.h

configure.ac

@@ -175,7 +175,7 @@ AX_CHECK_ALIGNED_ACCESS_REQUIRED
 # check for various headers
 AC_CHECK_HEADERS([execinfo.h sys/ioctl.h])
 
-### zlib (mandatory)
+### zlib
 CHECK_ZLIB()
 
 ### brotli and zstd content codings (optional)

m4/check_zlib.m4

@@ -1,33 +1,82 @@
 dnl @synopsis CHECK_ZLIB()
 dnl
-dnl Look for zlib. It is a hard requirement, not an option: the cache and the
-dnl WARC output are zip/gzip containers, and the bundled minizip calls zlib
-dnl directly. --with-zlib=DIR points at a non-standard prefix.
+dnl This macro searches for an installed zlib library. If nothing
+dnl was specified when calling configure, it searches first in /usr/local
+dnl and then in /usr. If the --with-zlib=DIR is specified, it will try
+dnl to find it in DIR/include/zlib.h and DIR/lib/libz.a. If --without-zlib
+dnl is specified, the library is not searched at all.
+dnl
+dnl If either the header file (zlib.h) or the library (libz) is not
+dnl found, the configuration exits on error, asking for a valid
+dnl zlib installation directory or --without-zlib.
+dnl
+dnl The macro defines the symbol HAVE_LIBZ if the library is found. You should
+dnl use autoheader to include a definition for this symbol in a config.h
+dnl file. Sample usage in a C/C++ source is as follows:
+dnl
+dnl   #ifdef HAVE_LIBZ
+dnl   #include <zlib.h>
+dnl   #endif /* HAVE_LIBZ */
+dnl
+dnl @version $Id$
+dnl @author Loic Dachary <loic@senga.org>
 dnl
-dnl Adds -lz to LIBS and defines HAVE_LIBZ.
 
-AC_DEFUN([CHECK_ZLIB], [
-AC_ARG_WITH([zlib],
-	[AS_HELP_STRING([--with-zlib=DIR],[root directory of the zlib installation])],
-	[zlib_want=$withval], [zlib_want=yes])
-if test "$zlib_want" = "no"; then
-	AC_MSG_ERROR([zlib cannot be disabled: the cache and the WARC output are zip/gzip containers, and the bundled minizip calls zlib directly])
+AC_DEFUN([CHECK_ZLIB],
+#
+# Handle user hints
+#
+[AC_MSG_CHECKING(if zlib is wanted)
+AC_ARG_WITH(zlib,
+[  --with-zlib=DIR root directory path of zlib installation [defaults to
+                    /usr/local or /usr if not found in /usr/local]
+  --without-zlib to disable zlib usage completely],
+[if test "$withval" != no ; then
+  AC_MSG_RESULT(yes)
+  ZLIB_HOME="$withval"
+else
+  AC_MSG_RESULT(no)
+fi], [
+AC_MSG_RESULT(yes)
+ZLIB_HOME=/usr/local
+if test ! -f "${ZLIB_HOME}/include/zlib.h"
+then
+        ZLIB_HOME=/usr
 fi
-if test "$zlib_want" != "yes"; then
-	# An explicit prefix is authoritative: if the header is not under it,
-	# error rather than silently pick a system copy.
-	if test ! -f "$zlib_want/include/zlib.h"; then
-		AC_MSG_ERROR([zlib requested at $zlib_want but $zlib_want/include/zlib.h is missing])
-	fi
-	CPPFLAGS="$CPPFLAGS -I$zlib_want/include"
-	LDFLAGS="$LDFLAGS -L$zlib_want/lib"
-elif test -f /usr/local/include/zlib.h; then
-	# Where the BSD ports tree lands zlib, and not always searched by default.
-	CPPFLAGS="$CPPFLAGS -I/usr/local/include"
-	LDFLAGS="$LDFLAGS -L/usr/local/lib"
-fi
-AC_CHECK_HEADER([zlib.h], [],
-	[AC_MSG_ERROR([zlib.h not found; install the zlib development files or pass --with-zlib=DIR])])
-AC_CHECK_LIB([z], [inflateEnd], [],
-	[AC_MSG_ERROR([libz not found; install the zlib development files or pass --with-zlib=DIR])])
+])
+
+#
+# Locate zlib, if wanted
+#
+if test -n "${ZLIB_HOME}"
+then
+        ZLIB_OLD_LDFLAGS=$LDFLAGS
+        ZLIB_OLD_CPPFLAGS=$LDFLAGS
+        LDFLAGS="$LDFLAGS -L${ZLIB_HOME}/lib"
+        CPPFLAGS="$CPPFLAGS -I${ZLIB_HOME}/include"
+        AC_LANG_SAVE
+        AC_LANG_C
+        AC_CHECK_LIB(z, inflateEnd, [zlib_cv_libz=yes], [zlib_cv_libz=no])
+        AC_CHECK_HEADER(zlib.h, [zlib_cv_zlib_h=yes], [zlib_cv_zlib_h=no])
+        AC_LANG_RESTORE
+        if test "$zlib_cv_libz" = "yes" -a "$zlib_cv_zlib_h" = "yes"
+        then
+                #
+                # If both library and header were found, use them
+                #
+                AC_CHECK_LIB(z, inflateEnd)
+                AC_MSG_CHECKING(zlib in ${ZLIB_HOME})
+                AC_MSG_RESULT(ok)
+        else
+                #
+                # If either header or library was not found, revert and bomb
+                #
+                AC_MSG_CHECKING(zlib in ${ZLIB_HOME})
+                LDFLAGS="$ZLIB_OLD_LDFLAGS"
+                CPPFLAGS="$ZLIB_OLD_CPPFLAGS"
+                AC_MSG_RESULT(failed)
+                AC_MSG_ERROR(either specify a valid zlib installation with --with-zlib=DIR or disable zlib usage with --without-zlib)
+        fi
+fi
+
 ])

src/htsback.c

@@ -48,6 +48,11 @@ Please visit our Website: http://www.httrack.com
 #include "htsftp.h"
 #include "htscodec.h"
 #include "htsproxy.h"
+#if HTS_USEZLIB
+#include "htszlib.h"
+#else
+#error HTS_USEZLIB not defined
+#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus

src/htscore.c

@@ -1980,9 +1980,10 @@ int httpmirror(char *url1, httrackp * opt) {
           }
 
           // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-          // A failed transfer has no body: r.adr holds debris from the aborted
-          // read, which would destroy the copy being re-fetched (#748).
-          if (r.statuscode > 0 && (r.adr != NULL || r.size == 0)) {
+          // An empty body must not overwrite the file when the transfer failed
+          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
+          // copy to 0 (#77 follow-up).
+          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {

src/htsglobal.h

@@ -138,11 +138,10 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DOSNAME 0
 #endif
 
-// zlib is mandatory: the cache is a zip and minizip calls it regardless
+// utiliser zlib?
 #ifndef HTS_USEZLIB
+// autoload
 #define HTS_USEZLIB 1
-#elif !HTS_USEZLIB
-#error HTS_USEZLIB=0 is not a supported configuration
 #endif
 
 // brotli and zstd content codings; off unless the build opted in (configure,

src/htsthread.c

@@ -44,18 +44,9 @@ Please visit our Website: http://www.httrack.com
 #endif
 #endif
 
-/* Outstanding threads, counted at spawn rather than by the child at entry, so
-   that a caller which spawns and immediately waits still joins them (#747). */
 static int process_chain = 0;
 static htsmutex process_chain_mutex = HTSMUTEX_INIT;
 
-static void process_chain_add(int delta) {
-  hts_mutexlock(&process_chain_mutex);
-  process_chain += delta;
-  assertf(process_chain >= 0);
-  hts_mutexrelease(&process_chain_mutex);
-}
-
 HTSEXT_API void htsthread_wait(void) {
   htsthread_wait_n(0);
 }
@@ -108,12 +99,20 @@ static void *hts_entry_point(void *tharg)
   void *const arg = s_args->arg;
   void (*fun) (void *arg) = s_args->fun;
 
-  freet(tharg);
+  free(tharg);
+
+  hts_mutexlock(&process_chain_mutex);
+  process_chain++;
+  assertf(process_chain > 0);
+  hts_mutexrelease(&process_chain_mutex);
 
   /* run */
   fun(arg);
 
-  process_chain_add(-1);
+  hts_mutexlock(&process_chain_mutex);
+  process_chain--;
+  assertf(process_chain >= 0);
+  hts_mutexrelease(&process_chain_mutex);
 #ifdef _WIN32
   return 0;
 #else
@@ -123,20 +122,18 @@ static void *hts_entry_point(void *tharg)
 
 /* create a thread */
 HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg) {
-  hts_thread_s *s_args = malloct(sizeof(hts_thread_s));
+  hts_thread_s *s_args = malloc(sizeof(hts_thread_s));
 
   assertf(s_args != NULL);
   s_args->arg = arg;
   s_args->fun = fun;
-  process_chain_add(1);
 #ifdef _WIN32
   {
     unsigned int idt;
     HANDLE handle =
       (HANDLE) _beginthreadex(NULL, 0, hts_entry_point, s_args, 0, &idt);
     if (handle == 0) {
-      process_chain_add(-1);
-      freet(s_args);
+      free(s_args);
       return -1;
     } else {
       /* detach the thread from the main process so that is can be independent */
@@ -154,8 +151,7 @@ HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg) {
         || pthread_attr_setstacksize(&attr, stackSize) != 0
         || (retcode =
             pthread_create(&handle, &attr, hts_entry_point, s_args)) != 0) {
-      process_chain_add(-1);
-      freet(s_args);
+      free(s_args);
       return -1;
     } else {
       /* detach the thread from the main process so that is can be independent */

src/httrack.c

@@ -306,8 +306,8 @@ int main(int argc, char **argv) {
     fprintf(stderr, "* %s\n", hts_errmsg(opt));
   }
   global_opt = NULL;
-  htsthread_wait(); /* pending threads still read opt */
   hts_free_opt(opt);
+  htsthread_wait();             /* wait for pending threads */
   hts_uninit();
 
 #ifdef _WIN32

tests/93_local-changes.test

@@ -94,7 +94,10 @@ expect_counts_match() {
     echo "OK"
 }
 lines() { printf '%s\n' "$@" | sort; }
-digest() { cksum <"$1" | tr -d '[:space:]'; }
+# A connection killed before the status line surfaces differently per platform
+# (macOS truncates the file, #748; Linux leaves it), so reset.bin is asserted on
+# its own and kept out of the exact lists.
+listed_but_reset() { listed "$1" | grep -v '/reset\.bin$' | sort; }
 
 # --- pass 1: nothing to compare against --------------------------------------
 httrack "${common[@]}" -O "$out" --purge-old=0 "${base}/changes/index.html" \
@@ -117,7 +120,6 @@ grep -aq "first crawl" "${out}/hts-log.txt" || {
 }
 
 host="127.0.0.1_${port}"
-resetdigest=$(digest "${out}/${host}/changes/reset.bin")
 # A leftover from some earlier failed attempt, at a name pass 2 fetches for the
 # first time: on disk, but never part of the previous mirror, so it is new.
 printf 'leftover junk' >"${out}/${host}/changes/fresh.html"
@@ -142,19 +144,21 @@ expect "gone is doomed.html and the old redirect target" \
 expect "changed is exactly the four moved resources" \
     "$(lines "${host}/changes/coded.bin" "${host}/changes/index.html" \
         "${host}/changes/moved.bin" "${host}/changes/moved.html")" \
-    "$(listed changed | sort)"
+    "$(listed_but_reset changed)"
 # Re-served byte for byte, 200, no Last-Modified, no ETag. flaky.bin gets there
 # through a failed transfer and a retry, so its file is notified twice;
 # redirtarget.html arrives behind a 302. sized.html has a byte-identical
 # payload, but the rewritten link to the renamed redirect target makes the file
-# on disk change length. reset.bin never completes a transfer (#746), so its
-# previous copy stands.
-expect "unchanged is exactly the seven stable resources" \
+# on disk change length.
+expect "unchanged is exactly the six stable resources" \
     "$(lines "${host}/changes/codedstable.bin" "${host}/changes/flaky.bin" \
-        "${host}/changes/redirtarget.html" "${host}/changes/reset.bin" \
-        "${host}/changes/sized.html" "${host}/changes/stable.bin" \
-        "${host}/changes/stable.html")" \
-    "$(listed unchanged | sort)"
+        "${host}/changes/redirtarget.html" "${host}/changes/sized.html" \
+        "${host}/changes/stable.bin" "${host}/changes/stable.html")" \
+    "$(listed_but_reset unchanged)"
+# reset.bin never completes a transfer, so it drops out of new.lst with its
+# mirrored copy still there. Whatever else it is, it is not a deletion.
+expect "a failed re-fetch is never reported gone" "" \
+    "$(listed gone | grep '/reset\.bin$' || true)"
 expect_counts_match "pass 2 counts match its lists"
 
 # Every mirrored file appears once and only once, across all four buckets.
@@ -188,15 +192,19 @@ test -f "${out}/${host}/changes/doomed.html" || {
     exit 1
 }
 echo "OK"
-expect "a failed transfer keeps its bytes" "$resetdigest" \
-    "$(digest "${out}/${host}/changes/reset.bin")"
+printf '[a failed transfer keeps its file] ..\t'
+test -f "${out}/${host}/changes/reset.bin" || {
+    echo "FAIL: reset.bin lost its previous copy"
+    exit 1
+}
+echo "OK"
 
 # --- pass 3: the same run with purging on ------------------------------------
 httrack "${common[@]}" -O "$out" --update "${base}/changes/index.html" \
     >"${tmpdir}/log3" 2>&1
 expect "the report says files were purged" "true" "$(field purged)"
 expect "gone is the page that only pass 2 linked" \
-    "${host}/changes/transient.html" "$(listed gone | sort)"
+    "${host}/changes/transient.html" "$(listed_but_reset gone)"
 printf '[a purged file is off disk] ..\t'
 test ! -f "${out}/${host}/changes/transient.html" || {
     echo "FAIL: transient.html survived --purge-old"

tests/96_local-refetch-keep.test

@@ -1,28 +0,0 @@
-#!/bin/bash
-#
-# A re-fetch cut mid-header must leave the mirrored file alone (#748): unfixed,
-# the aborted read's leftover buffer lands on it. err.bin is the control, an
-# HTTP 500 on the same resource already keeping its copy; stay.bin answers
-# normally with a new body, so a fix that stopped overwriting anything fails.
-#
-# Purging is off: an update purge deletes both files for an unrelated reason
-# (#746), which would hide what this asserts.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --rerun-args '--update --purge-old=0' \
-    --log-found 'after 2 retries at link .*keep/page\.html' \
-    --log-found 'after 2 retries at link .*keep/data\.bin' \
-    --file-matches 'keep/page.html' 'KEEP-PAGE-V1' \
-    --file-not-matches 'keep/page.html' 'HTTP/1\.0 200' \
-    --file-matches 'keep/data.bin' 'KEEP-BIN-V1' \
-    --file-not-matches 'keep/data.bin' 'HTTP/1\.0 200' \
-    --file-min-bytes 'keep/data.bin' 2048 \
-    --file-matches 'keep/err.bin' 'KEEP-ERR-V1' \
-    --file-min-bytes 'keep/err.bin' 2048 \
-    --file-matches 'keep/stay.bin' 'KEEP-STAY-V2' \
-    --file-min-bytes 'keep/stay.bin' 2048 \
-    httrack 'BASEURL/keep/index.html' --retries=2

tests/Makefile.am

@@ -194,7 +194,6 @@ TESTS = \
 	92_local-proxytrack-ndx-fields.test \
 	93_local-changes.test \
 	94_local-single-file.test \
-	95_local-sitemap.test \
-	96_local-refetch-keep.test
+	95_local-sitemap.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -1000,62 +1000,6 @@ class Handler(SimpleHTTPRequestHandler):
         v = 1 if self.refetch_pass() == 1 else 2
         self.send_raw(b"<html><body><p>STAY-V%d</p></body></html>" % v, "text/html")
 
-    # --- re-fetch cut mid-header, so nothing is stored (#746, #748) ---------
-    KEEP_PAGE = b"<html><body><p>KEEP-PAGE-V1</p></body></html>"
-    KEEP_BIN = b"KEEP-BIN-V1\n" + b"\x51\x52\x53\x54" * 512
-    KEEP_ERR = b"KEEP-ERR-V1\n" + b"\x61\x62\x63\x64" * 512
-
-    def send_cut_headers(self):
-        """Hang up mid-header: the response never becomes parseable."""
-        self.close_connection = True
-        try:
-            self.wfile.write(b"HTTP/1.0 200 OK\r\nContent-Ty")
-            self.wfile.flush()
-        except OSError:
-            pass
-        self.connection.close()
-
-    def route_keep_index(self):
-        self.refetch_pass()
-        self.send_html(
-            '\t<a href="page.html">page</a>\n'
-            '\t<a href="data.bin">data</a>\n'
-            '\t<a href="err.bin">err</a>\n'
-            '\t<a href="stay.bin">stay</a>\n'
-        )
-
-    def route_keep_page(self):
-        if self.refetch_pass() == 1:
-            self.send_raw(self.KEEP_PAGE, "text/html")
-        else:
-            self.send_cut_headers()
-
-    def route_keep_data(self):
-        if self.refetch_pass() == 1:
-            self.send_raw(self.KEEP_BIN, "application/octet-stream")
-        else:
-            self.send_cut_headers()
-
-    # Control: an HTTP error on the same resource already keeps the copy, so
-    # the cut-header routes above must end up indistinguishable from it.
-    def route_keep_err(self):
-        if self.refetch_pass() == 1:
-            self.send_raw(self.KEEP_ERR, "application/octet-stream")
-        else:
-            self.send_response(500)
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", "0")
-            self.end_headers()
-
-    # Control: answers normally, with a new body on pass 2, so a fix that
-    # stopped overwriting mirrored files altogether would be caught.
-    def route_keep_stay(self):
-        v = 1 if self.refetch_pass() == 1 else 2
-        self.send_raw(
-            b"KEEP-STAY-V%d\n" % v + b"\x71\x72\x73\x74" * 512,
-            "application/octet-stream",
-        )
-
     # Echo what httrack advertised, so a crawl can assert the header.
     def route_codec_ae(self):
         self.send_raw(
@@ -2006,11 +1950,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/uptrunc/page.html": route_uptrunc_page,
         "/uptrunc/file.bin": route_uptrunc_file,
         "/uptrunc/stay.html": route_uptrunc_stay,
-        "/keep/index.html": route_keep_index,
-        "/keep/page.html": route_keep_page,
-        "/keep/data.bin": route_keep_data,
-        "/keep/err.bin": route_keep_err,
-        "/keep/stay.bin": route_keep_stay,
         "/types/index.html": route_types_index,
         "/types/control.php": route_types,
         "/types/photo.png": route_types,