prototype: format the touched lines

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <xroche@gmail.com>

.gitattributes

@@ -6,6 +6,3 @@
 # bash die on $'\r' on every line of them.
 *.test text eol=lf
 *.sh text eol=lf
-
-# Scoped to this file so a union can never touch build logic.
-tests/tests-list.mk merge=union

.github/workflows/ci.yml

@@ -46,32 +46,10 @@ jobs:
       - name: Configure
         run: |
           set -euo pipefail
-          # Regenerate: configure and the Makefile.in's are not tracked.
+          # Regenerate from configure.ac/Makefile.am to validate them; the
+          # committed generated files already let a plain checkout build.
           autoreconf -fi
-          # Disabling zlib must fail here rather than at link with a pile of
-          # undefined minizip references (#735). Both spellings, so a rewrite
-          # cannot keep one and lose the other. Probed out-of-tree to leave
-          # nothing behind.
-          nozlib="$RUNNER_TEMP/nozlib"
-          for arg in --without-zlib --with-zlib=no; do
-            rm -rf "$nozlib" && mkdir -p "$nozlib"
-            if (cd "$nozlib" && "$GITHUB_WORKSPACE/configure" "$arg" >out.log 2>&1); then
-              echo "::error::configure $arg succeeded; it must be rejected"
-              exit 1
-            fi
-            # ... and for the stated reason, not an unrelated configure failure.
-            grep -q "zlib cannot be disabled" "$nozlib/out.log" \
-              || { cat "$nozlib/out.log"; exit 1; }
-          done
           ./configure
-          # Same dead end from the compile side. The bare compile is the
-          # control: without it a broken probe would pass vacuously.
-          hdr='#include "htsglobal.h"'
-          echo "$hdr" | $CC -I. -Isrc -fsyntax-only -xc -
-          if echo "$hdr" | $CC -DHTS_USEZLIB=0 -I. -Isrc -fsyntax-only -xc - 2>/dev/null; then
-            echo "::error::-DHTS_USEZLIB=0 compiled; htsglobal.h must reject it"
-            exit 1
-          fi
           # a missing decoder would silently drop the coding from Accept-Encoding
           grep -q "define HTS_USEBROTLI 1" config.h
           grep -q "define HTS_USEZSTD 1" config.h
@@ -79,10 +57,7 @@ jobs:
       - name: Build
         run: make -j"$(nproc)"
 
-      # A backstop only: tests/test-timeout.sh bounds each test, and a healthy run
-      # is a minute here, two on macOS. Without it a stall ran to the job's 6h default.
       - name: Test
-        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -122,7 +97,6 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test without python3
-        timeout-minutes: 20
         run: |
           set -euo pipefail
           # Hide every python3* so `command -v python3` fails like it does in the
@@ -177,7 +151,6 @@ jobs:
           sudo ifconfig lo0 alias 127.0.0.3 up
 
       - name: Test
-        timeout-minutes: 20
         run: |
           jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -250,7 +223,6 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -308,7 +280,6 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -352,7 +323,6 @@ jobs:
       - name: Test (offline self-tests under MSan)
         env:
           MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
-        timeout-minutes: 20
         run: |
           set -euo pipefail
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
@@ -434,7 +404,6 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -627,12 +596,7 @@ jobs:
           set -euo pipefail
           git fetch --no-tags origin \
             "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
-          # Merge base, not the branch tip: master moves during a run, and a
-          # tip-relative diff blames its new C on this PR.
-          if ! base="$(git merge-base "origin/${{ github.base_ref }}" HEAD)"; then
-            echo "::error::no merge base with origin/${{ github.base_ref }}; cannot scope the check."
-            exit 1
-          fi
+          base="origin/${{ github.base_ref }}"
           set +e
           diff="$(git clang-format --binary clang-format-19 --style=file \
                     --diff --extensions c,h "$base")"
@@ -674,8 +638,6 @@ jobs:
           git fetch --no-tags origin \
             "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
           base="origin/${{ github.base_ref }}"
-          # Three dots: merge-base scoped, so commits landing on master mid-run
-          # are not counted as this PR's.
           changed="$(git diff --name-only "$base"...HEAD)"
           has() { printf '%s\n' "$changed" | grep -qx "$1"; }
           if has man/httrack.1 && ! has html/httrack.man.html; then

.github/workflows/windows-build.yml

@@ -189,55 +189,25 @@ jobs:
           # A wedged crawl must not eat the job's timeout budget. timeout(1)'s
           # signals can't reap a native httrack.exe (MSYS signals don't reach it),
           # so a hang orphaned processes that starved the runner; run_with_timeout
-          # TerminateProcess-es the whole tree. 600s is unchanged: it clears the
-          # 540s a three-pass crawl may legitimately take under local-crawl.sh's
-          # own watchdogs, against a slowest healthy test here of 39s.
+          # TerminateProcess-es the whole tree. 600s clears the slowest multi-pass
+          # crawl (a few passes at --max-time=120 each).
           . ./testlib.sh
-          per_test=600
 
-          # The whole suite must give up before the step timeout above. A cancelled
-          # step keeps neither its log nor the artifacts the later if:always()
-          # steps would upload, so an overrun that ends in a cancel tells us
-          # nothing; failing on our own terms keeps both. Healthy runs take 8-9
-          # min. The check sits between tests, so the step can still reach 25 min
-          # plus one per-test budget, and that worst case stays inside the 45.
-          suite_deadline=1500
-          started=$SECONDS
-
-          # Survives into the artifact even if the tail of the step log does not.
-          progress=suite-progress.log
-          : >"$progress"
-
-          pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
-          # Globbed, not enumerated: a new NNN_engine-*.test or NNN_local-*.test
-          # is picked up automatically instead of silently getting zero coverage.
-          for t in 00_runnable.test *_engine-*.test *_zlib-*.test \
-                   *_local-*.test *_watchdog*.test *_crawl_proxy_https.test \
-                   *_crawl-log-salvage.test; do
-            elapsed=$((SECONDS - started))
-            if [ "$elapsed" -ge "$suite_deadline" ]; then
-              echo "::error::suite deadline: ${elapsed}s elapsed, stopping before $t"
-              echo "DEADLINE before $t after ${elapsed}s" >>"$progress"
-              # Per-test start times, so the slow ones are named rather than guessed.
-              sed 's/^/      /' "$progress"
-              deadline=1
-              break
-            fi
-            echo "RUN $t at ${elapsed}s" >>"$progress"
+          pass=0 fail=0 skip=0 failed="" skipped=""
+          for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
+                   *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
+                   60_crawl-log-salvage.test; do
             rc=0
-            # Same guard "make check" uses on POSIX, so a wedge is diagnosed the
-            # same way on every platform. It dumps before it kills, which a bare
-            # run_with_timeout cannot: by the time that returns, the tree whose
-            # stack we wanted is already gone.
-            HTTRACK_TEST_TIMEOUT=$per_test bash ./test-timeout.sh "$t" >"$t.log" 2>&1 || rc=$?
+            run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
               77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
               124)
                 fail=$((fail + 1)) failed="$failed $t"
-                # test-timeout.sh has already written the process list, the stacks
-                # and the killed crawl's own logs into $t.log.
                 echo "FAIL $t (timed out, tree killed)"
+                # Re-running a wedge traced would just hang again: salvage the
+                # killed crawl's own logs into the artifact instead.
+                dump_crawl_logs >>"$t.log"
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
               *)
@@ -245,53 +215,23 @@ jobs:
                 echo "FAIL $t (exit $rc)"
                 # These assert with `test "$(...)" == "..." || exit 1`, which
                 # says nothing at all on failure. Re-run traced, still bounded.
-                run_with_timeout "$per_test" bash -x "$t" >>"$t.log" 2>&1 || true
+                run_with_timeout 600 bash -x "$t" >>"$t.log" 2>&1 || true
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
             esac
-            echo "$rc $t" >>"$progress"
-            # An orphaned native httrack.exe spins and starves the runner, which
-            # is how this job dies with "lost communication" rather than a plain
-            # timeout. Clear them between tests and name whoever leaked them.
-            reap_leftover_processes "$t" | tee -a "$progress"
           done
           echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
             tee -a "$GITHUB_STEP_SUMMARY"
 
           # Every gate here exits 77, so an all-skipped suite would report green having
           # tested nothing: pin the skips, and floor the passes in case the glob empties.
-          # One name per line, so two branches each appending one don't collide on the
-          # same line; compared as a sorted set below, so glob discovery order can't
-          # cause a false mismatch either.
-          # footer-overflow and purge-longpath skip on Windows (need a path past MAX_PATH);
-          # crange pending #581;
-          # webdav-default and webdav-mime need a reapable background listener, which MSYS cannot give them;
+          # footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
+          # webdav-mime needs a reapable background listener, which MSYS cannot give it;
           # badmtime needs a filesystem that stores an mtime past gmtime's range;
-          # single-file ends on a GUI half needing htsserver, which this job does not build;
-          # update-304-leak needs a LeakSanitizer build, which MSVC has no equivalent of;
-          # crash-symbolize needs backtrace(), which Windows has no equivalent of.
-          expected_skips="01_engine-footer-overflow.test
-          100_local-purge-longpath.test
-          114_local-update-304-leak.test
-          120_local-proxytrack-webdav-default.test
-          48_local-crange-memresume.test
-          71_local-crange-repaircache.test
-          79_local-proxytrack-webdav-mime.test
-          80_engine-crash-symbolize.test
-          88_local-proxytrack-badmtime.test
-          94_local-single-file.test"
-          # First, or the deadline reads as an unexplained shortfall in the gates below.
-          [ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
+          # single-file ends on a GUI half needing htsserver, which this job does not build.
+          expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
-          # Word-split on whitespace (space-joined $skipped, newline-joined
-          # expected_skips both work) and sort, so the compare is a set, not a string.
-          got=$(printf '%s\n' $skipped | sort)
-          want=$(printf '%s\n' $expected_skips | sort)
-          if [ "$got" != "$want" ]; then
-            echo "::error::skip set changed from expected; - missing, + newly skipped"
-            diff -u <(echo "$want") <(echo "$got") | tail -n +3 | sed 's/^/      /'
-            exit 1
-          fi
+          [ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
           [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
 
       - name: Upload the test logs

AGENTS.md

@@ -19,25 +19,19 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
   `-c16` bigcrawl the way Python's default backlog of 5 did.
   Or run `sh build.sh` to do bootstrap + configure + make in one shot.
-- A `tests/NN_*.test` runs only if listed in `tests/tests-list.mk`; an
+- A `tests/NN_*.test` runs only if listed in `tests/Makefile.am`'s `TESTS`; an
   unregistered file is silently skipped.
 - `make check` prepends the build's `src/` to `PATH`, but a hand-run `.test` does
   not — an installed `/usr/bin/httrack` then shadows your build. Run via `make
   check`, or `PATH="<bld>/src:$PATH"` for a manual run.
 - Give new `.test` scripts `set -e`: the older ones predate the rule, so several
   `local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
-- Run teardown with errexit off: `trap 'set +e; cleanup' EXIT`. Under `set -e` a
-  failing cleanup command becomes the test's exit status (#773). Keep the other
-  signals on their own `trap` line, or errexit stays off for the rest of the run.
-  The guard also resets `$?`, so save it first if teardown reads it.
-- Never pipe into `grep -q`: it exits on the first match, so whatever the
-  producer had left to write takes SIGPIPE, and under `pipefail` that becomes
-  the pipeline's status. `cmd | grep -q M && fail` then never fires and a probe
-  that proved nothing reads as "marker absent"; `cmd | grep -q M || fail` fails
-  a test whose marker was present. bash issues one `write()` per line, so any
-  match that is not on the last line is exposed. Capture the reply, assert the
-  status line it must carry (an empty, truncated or redirected one is
-  marker-free too), then match with a here-string: `grep -q M <<<"$reply"`.
+- Never assert with `cmd | grep -q MARKER && fail`. Under `pipefail` the
+  pipeline is non-zero both when `cmd` fails and when `grep -q` matches early
+  and SIGPIPEs it, so the `&&` never fires and a probe that proved nothing reads
+  as "marker absent". Capture the reply, assert the status line it must carry
+  (an empty, truncated or redirected one is marker-free too), then match with a
+  here-string.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

configure.ac

@@ -175,7 +175,7 @@ AX_CHECK_ALIGNED_ACCESS_REQUIRED
 # check for various headers
 AC_CHECK_HEADERS([execinfo.h sys/ioctl.h])
 
-### zlib (mandatory)
+### zlib
 CHECK_ZLIB()
 
 ### brotli and zstd content codings (optional)

fuzz/Makefile.am

@@ -2,7 +2,7 @@
 if FUZZERS
 noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
 	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx \
-	fuzz-htsparse fuzz-singlefile fuzz-sitemap
+	fuzz-htsparse fuzz-singlefile
 endif
 
 AM_CPPFLAGS = \
@@ -28,7 +28,6 @@ fuzz_header_SOURCES = fuzz-header.c fuzz.h
 fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
 fuzz_htsparse_SOURCES = fuzz-htsparse.c fuzz.h
 fuzz_singlefile_SOURCES = fuzz-singlefile.c fuzz.h
-fuzz_sitemap_SOURCES = fuzz-sitemap.c fuzz.h
 
 # List corpus files explicitly: automake does not expand EXTRA_DIST globs.
 EXTRA_DIST = README.md run-fuzzers.sh \
@@ -53,6 +52,4 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/singlefile/img-src.html corpus/singlefile/link-rel.html \
 	corpus/singlefile/style-block.html corpus/singlefile/style-attr.html \
 	corpus/singlefile/srcset.html corpus/singlefile/rawtext.html \
-	corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html \
-	corpus/sitemap/urlset.xml corpus/sitemap/sitemapindex.xml \
-	corpus/sitemap/truncated.xml corpus/sitemap/urlset.xml.gz
+	corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html

fuzz/corpus/sitemap/sitemapindex.xml

@@ -1 +0,0 @@
-<sitemapindex><sitemap><loc>http://h.test/s2.xml.gz</loc></sitemap></sitemapindex>

fuzz/corpus/sitemap/truncated.xml

@@ -1 +0,0 @@
-<urlset><loc>http://h.test/x

fuzz/corpus/sitemap/urlset.xml

@@ -1 +0,0 @@
-
<?xml version="1.0"?><urlset><url><loc>http://h.test/a.html</loc></url><url><loc>https://h.test/b?x=1&amp;y=2</loc></url></urlset>

fuzz/fuzz-sitemap.c

@@ -1,60 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the sitemap <loc> scanner (htssitemap.c): raw XML, gzip-framed bodies
-   and truncated streams all arrive here straight off the network. */
-#include "fuzz.h"
-#include "htssitemap.h"
-
-static hts_boolean sm_count(void *arg, const char *url) {
-  int *const n = (int *) arg;
-
-  (void) url;
-  (*n)++;
-  return HTS_TRUE;
-}
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  static const int caps[] = {0, 1, 16, HTS_SITEMAP_MAX_URLS_DOC};
-  hts_boolean is_index;
-  char *body;
-  int n = 0, cap;
-
-  if (size == 0)
-    return 0;
-  cap = caps[data[0] % (sizeof(caps) / sizeof(caps[0]))];
-  data++, size--;
-  /* A heap copy of exactly `size` bytes: the scanner must never rely on a
-     terminator, and ASan turns any overread into a report. */
-  body = malloct(size != 0 ? size : 1);
-  memcpy(body, data, size);
-
-  (void) hts_sitemap_scan(body, size, cap, &is_index, sm_count, &n);
-
-  freet(body);
-  return 0;
-}

html/cmdguide.html

@@ -163,26 +163,8 @@ the index" problems disappear.</p>
   <tr><td><tt>--near (-n)</tt></td><td>Also fetch non-HTML files "near" a followed link, such as an image linked from a page you kept but hosted elsewhere.</td></tr>
   <tr><td><tt>--ext-depth (-%e)</tt></td><td>How many levels of external links to follow once the crawl leaves your scope (default 0).</td></tr>
   <tr><td><tt>--test (-t)</tt></td><td>Also HEAD-test links that fall outside the scope, which are normally refused, without downloading them: a way to see what scope is excluding.</td></tr>
-  <tr><td><tt>--sitemap (-%m), --sitemap-url URL (-%mu)</tt></td><td>Also take start URLs from the site's sitemap, for pages nothing links to. Off by default.</td></tr>
 </table>
 
-<p>Link-following only finds what something links to. Anything a site publishes
-solely in its sitemap is invisible to HTTrack unless you ask for it.
-<tt>--sitemap</tt> reads the start host's <tt>robots.txt</tt> for
-<tt>Sitemap:</tt> lines and falls back to <tt>/sitemap.xml</tt>;
-<tt>--sitemap-url</tt> names one directly. Nested <tt>sitemapindex</tt> files
-and gzipped <tt>.xml.gz</tt> sitemaps are followed. The URLs found become start
-URLs with the full depth budget, but they still go through your filters and
-scope rules, so a sitemap cannot widen a crawl you deliberately narrowed. It is
-off by default because a sitemap can list thousands of pages nothing links
-to.</p>
-
-<p>One surprise worth knowing: a sitemap you name with <tt>--sitemap-url</tt>,
-and one the site itself declares in <tt>robots.txt</tt>, are fetched even when
-<tt>robots.txt</tt> disallows that path, because naming or declaring a sitemap
-is an invitation to read it. Only the guessed <tt>/sitemap.xml</tt> obeys a
-<tt>Disallow</tt>. The URLs listed inside are gated normally either way.</p>
-
 <p>The single most common surprise is "only the home page came down." That is
 usually not a scope option at all: it is an off-host redirect. A start URL of
 <tt>http://example.com/</tt> that redirects to <tt>https://www.example.com/</tt>

html/fcguide.html

@@ -237,7 +237,7 @@ Build options:
   x  replace external html links by error pages (--replace-external)
  %x  do not include any password for external password protected websites (%x0 include) (--no-passwords)
  %q *include query string for local files (useless, for information purpose only) (%q0 don't include) (--include-query-string)
-  o *save the server's error pages (404..) (o0 discard them) (--generate-errors)
+  o *generate output html file in case of error (404..) (o0 don't generate) (--generate-errors)
   X *purge old files after update (X0 keep delete) (--purge-old[=N])
 
 Spider options:
@@ -413,7 +413,7 @@ site. Specifically, the defauls are:
   NN  name conversion type (0 *original structure, 1+: see below)
   LN  long names (L1 *long names / L0 8-3 conversion)
   K   keep original links (e.g. http://www.adr/link) (K0 *relative link)
-  o *save the server's error pages (404..) (o0 discard them)
+  o *generate output html file in case of error (404..) (o0 don't generate)
   X *purge old files after update (X0 keep delete)
   bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
   u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
@@ -473,11 +473,11 @@ store them with the same names used on the web site.
 URLs within this web site are adjusted to point to the files in the
 mirror.
 
-<pre><b><i>  o *save the server's error pages (404..) (o0 discard them) </i></b></pre>
+<pre><b><i>  o *generate output html file in case of error (404..) (o0 don't generate) </i></b></pre>
 
-<p align=justify> IF a page cannot be downloaded, the error page the
-server sent is saved in its place, so a broken link still lands on the
-site's own 'not found' page.  This makes browsing go a lot smoother.
+<p align=justify> IF there are errors in downloading, create a file that
+indicates that the URL was not found.  This makes browsing go a lot
+smoother.
 
 <pre><b><i>  X *purge old files after update (X0 keep delete) </i></b></pre>
 
@@ -1011,7 +1011,7 @@ Build options:
   LN  long names (L1 *long names / L0 8-3 conversion)
   K   keep original links (e.g. http://www.adr/link) (K0 *relative link)
   x   replace external html links by error pages
-  o *save the server's error pages (404..) (o0 discard them)
+  o *generate output html file in case of error (404..) (o0 don't generate)
   X *purge old files after update (X0 keep delete)
   %x  do not include any password for external password protected websites (%x0 include) (--no-passwords)
   %q *include query string for local files (information only) (%q0 don't include) (--include-query-string)
@@ -1118,9 +1118,9 @@ deactivated byt his process.
 httrack http://www.shoesizes.com -O /tmp/shoesizes -x
 </i></b></pre>
 
-<p align=justify> This option keeps the server's '404' error pages out
-of the mirror, even though there were URLs pointing to the missing
-files.  It is useful for saving space as well as eliminating
+<p align=justify> This option prevents the generation of '404' error
+files to replace files that were not found even though there were URLs
+pointing to them.  It is useful for saving space as well as eliminating
 unnecessary files in operations where a working web site is not the
 desired result. 
 

html/httrack.man.html

@@ -87,8 +87,8 @@ offline browser : copy websites to a local directory</p>
 --host-control[=N]</b> ] [ <b>-%P,
 --extended-parsing[=N]</b> ] [ <b>-n, --near</b> ] [ <b>-t,
 --test</b> ] [ <b>-%L, --list</b> ] [ <b>-%S, --urllist</b>
-] [ <b>-%m, --sitemap</b> ] [ <b>-NN, --structure[=N]</b> ]
-[ <b>-%N, --delayed-type-check</b> ] [ <b>-%D,
+] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
+--delayed-type-check</b> ] [ <b>-%D,
 --cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
 ] [ <b>-%Z, --single-file</b> ] [ <b>-LN,
 --long-names[=N]</b> ] [ <b>-KN, --keep-links[=N]</b> ] [
@@ -577,22 +577,6 @@ URL per line) (--list &lt;param&gt;)</p></td></tr>
 
 <p>&lt;file&gt; add all scan rules located in this text
 file (one scan rule per line) (--urllist &lt;param&gt;)</p></td></tr>
-<tr valign="top" align="left">
-<td width="9%"></td>
-<td width="4%">
-
-
-<p>-%m</p></td>
-<td width="5%"></td>
-<td width="82%">
-
-
-<p>seed the crawl from the site&rsquo;s sitemap (robots.txt
-Sitemap:, then /sitemap.xml); --sitemap-url URL names one
-explicitly. A sitemap you name, or one the site declares, is
-fetched even under robots.txt Disallow; only the guessed
-/sitemap.xml obeys it. The URLs found still pass every
-filter and scope rule (--sitemap)</p></td></tr>
 </table>
 
 <h3>Build options:
@@ -777,8 +761,8 @@ information purpose only) (%q0 don&rsquo;t include)
 <td width="82%">
 
 
-<p>*save the server&rsquo;s error pages (404..) (o0 discard
-them) (--generate-errors)</p></td></tr>
+<p>*generate output html file in case of error (404..) (o0
+don&rsquo;t generate) (--generate-errors)</p></td></tr>
 <tr valign="top" align="left">
 <td width="9%"></td>
 <td width="4%">

html/server/option8.html

@@ -108,7 +108,6 @@ ${do:end-if}
 <input type="hidden" name="keepqueryorder" value="">
 <input type="hidden" name="toler" value="">
 <input type="hidden" name="http10" value="">
-<input type="hidden" name="sitemap" value="">
 
 <input type="checkbox" name="cookies" ${checked:cookies}
 			title='${html:LANG_I1b}' onMouseOver="info('${html:LANG_I1b}'); return true" onMouseOut="info('&nbsp;'); return true"
@@ -144,17 +143,6 @@ ${listid:robots:LISTDEF_8}
 </select>
 <br><br>
 
-<input type="checkbox" name="sitemap" ${checked:sitemap}
-			title='${html:LANG_SITEMAPTIP}' onMouseOver="info('${html:LANG_SITEMAPTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
-> ${LANG_SITEMAP}
-<br><br>
-
-${LANG_SITEMAPURL}
-<input name="sitemapurl" value="${sitemapurl}" size="40"
-			title='${html:LANG_SITEMAPURLTIP}' onMouseOver="info('${html:LANG_SITEMAPURLTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
->
-<br><br>
-
 <input type="checkbox" name="updhack" ${checked:updhack}
 			title='${html:LANG_I1k}' onMouseOver="info('${html:LANG_I1k}'); return true" onMouseOut="info('&nbsp;'); return true"
 > ${LANG_I62b}

html/server/step2.html

@@ -141,8 +141,6 @@ ${do:copy:KeepSlashes:keepslashes}
 ${do:copy:KeepQueryOrder:keepqueryorder}
 ${do:copy:StripQuery:stripquery}
 ${do:copy:StoreAllInCache:cache2}
-${do:copy:Sitemap:sitemap}
-${do:copy:SitemapUrl:sitemapurl}
 ${do:copy:Warc:warc}
 ${do:copy:WarcFile:warcfile}
 ${do:copy:Changes:changes}

html/server/step4.html

@@ -188,8 +188,6 @@ ${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one *
 	${test:toler:--tolerant}
 	${test:http10:--http-10}
 	${test:cache2:--store-all-in-cache}
-	${test:sitemap:--sitemap}
-	${test:sitemapurl:--sitemap-url "}${html:sitemapurl}${test:sitemapurl:"}
 	${test:warc:--warc}
 	${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
 	${test:changes:--changes}
@@ -245,8 +243,6 @@ KeepSlashes=${ztest:keepslashes:0:1}
 KeepQueryOrder=${ztest:keepqueryorder:0:1}
 StripQuery=${stripquery}
 StoreAllInCache=${ztest:cache2:0:1}
-Sitemap=${ztest:sitemap:0:1}
-SitemapUrl=${sitemapurl}
 Warc=${ztest:warc:0:1}
 WarcFile=${warcfile}
 Changes=${ztest:changes:0:1}

html/step9_opt5.html

@@ -122,7 +122,7 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
   </small><br><br>
 <!-- -->
   <li>No error pages</li>
-  <br><small>Do not save the error pages sent by the server (if a 404 error occurred, for example)
+  <br><small>Do not generate error pages (if a 404 error occurred, for example)
   <br>If a page is missing on the remote site, there will not be any warning on the local site
   </small><br><br>
 <!-- -->

lang.def

@@ -1054,11 +1054,3 @@ LANG_SINGLEFILEMAX
 Largest inlined asset (bytes):
 LANG_SINGLEFILEMAXTIP
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
-LANG_SITEMAP
-Seed the crawl from the site's sitemap
-LANG_SITEMAPTIP
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-LANG_SITEMAPURL
-Sitemap address:
-LANG_SITEMAPURLTIP
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.

lang/Bulgarian.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Íàé-ãîëÿì âãðàäåí ðåñóðñ (áàéòîâå):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Ðåñóðñ íàä òîçè ðàçìåð çàïàçâà îáèêíîâåíà âðúçêà; îñòàâåòå ïðàçíî çà ñòîéíîñòòà ïî ïîäðàçáèðàíå îò 10485760 áàéòà.
-Seed the crawl from the site's sitemap
-Çàïî÷âàíå íà îáõîæäàíåòî îò êàðòàòà íà ñàéòà
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Ïðî÷èòàíå íà êàðòàòà íà ñàéòà (ðåäîâåòå Sitemap: â robots.txt, ñëåä òîâà /sitemap.xml) è äîáàâÿíå íà âñåêè ïîñî÷åí URL àäðåñ êàòî íà÷àëåí.
-Sitemap address:
-Àäðåñ íà êàðòàòà íà ñàéòà:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Àäðåñ íà êàðòà íà ñàéòà, êîÿòî äà áúäå ïðî÷åòåíà âìåñòî ñîíäèðàíå íà ñàéòà; îñòàâåòå ïðàçíî, çà äà ñå ïðîâåðè robots.txt, ñëåä òîâà /sitemap.xml.

lang/Castellano.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Tamaño máximo del recurso incrustado (bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Un recurso mayor que este tamaño conserva un enlace normal; déjelo vacío para el valor predeterminado de 10485760 bytes.
-Seed the crawl from the site's sitemap
-Iniciar el rastreo desde el mapa del sitio
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Leer el mapa del sitio (líneas Sitemap: de robots.txt, luego /sitemap.xml) y añadir como dirección inicial cada URL que incluya.
-Sitemap address:
-Dirección del mapa del sitio:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Dirección de un mapa del sitio que leer en lugar de sondear el sitio; déjelo vacío para sondear robots.txt y luego /sitemap.xml.

lang/Cesky.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Nejvìtší vložený zdroj (bajty):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Zdroj vìtší než tato velikost si ponechá bìžný odkaz; ponechte prázdné pro výchozí hodnotu 10485760 bajtù.
-Seed the crawl from the site's sitemap
-Zahájit procházení z mapy webu
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Naèíst mapu webu (øádky Sitemap: v souboru robots.txt, poté /sitemap.xml) a pøidat každou uvedenou adresu URL jako výchozí.
-Sitemap address:
-Adresa mapy webu:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adresa mapy webu, která se má naèíst místo zjiš<69>ování na webu; ponechte prázdné pro zjištìní z robots.txt a poté /sitemap.xml.

lang/Chinese-BIG5.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 內嵌資源大小上限（位元組）：
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 超過此大小的資源會保留一般連結；留空則使用預設的 10485760 位元組。
-Seed the crawl from the site's sitemap
-從網站的 Sitemap 開始擷取
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-讀取網站的 Sitemap（robots.txt 中的 Sitemap: 行，然後 /sitemap.xml），並將其中列出的每個網址加入為起始網址。
-Sitemap address:
-Sitemap 位址：
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-要讀取的 Sitemap 位址，用來取代自動探測；留空則先探測 robots.txt 再探測 /sitemap.xml。

lang/Chinese-Simplified.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 内嵌资源大小上限（字节）：
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 超过此大小的资源会保留普通链接；留空则使用默认的 10485760 字节。
-Seed the crawl from the site's sitemap
-从网站的 Sitemap 开始抓取
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-读取网站的 Sitemap（robots.txt 中的 Sitemap: 行，然后 /sitemap.xml），并将其中列出的每个网址添加为起始网址。
-Sitemap address:
-Sitemap 地址：
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-要读取的 Sitemap 地址，用来代替自动探测；留空则先探测 robots.txt 再探测 /sitemap.xml。

lang/Croatian.txt

@@ -978,11 +978,3 @@ Largest inlined asset (bytes):
 Najveæi ugraðeni resurs (bajtovi):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Resurs veæi od ove velièine zadr¾ava obiènu poveznicu; ostavite prazno za zadanih 10485760 bajtova.
-Seed the crawl from the site's sitemap
-Pokreni pretra¾ivanje iz karte web-mjesta
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Proèitaj kartu web-mjesta (retke Sitemap: iz robots.txt, zatim /sitemap.xml) i dodaj svaki navedeni URL kao poèetnu adresu.
-Sitemap address:
-Adresa karte web-mjesta:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adresa karte web-mjesta koju treba proèitati umjesto ispitivanja web-mjesta; ostavite prazno za ispitivanje robots.txt pa /sitemap.xml.

lang/Dansk.txt

@@ -1024,11 +1024,3 @@ Largest inlined asset (bytes):
 Største indlejrede ressource (byte):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 En ressource over denne størrelse beholder et almindeligt link; lad feltet stå tomt for standardværdien på 10485760 byte.
-Seed the crawl from the site's sitemap
-Start gennemgangen fra webstedets sitemap
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Læs webstedets sitemap (Sitemap:-linjer i robots.txt, derefter /sitemap.xml) og tilføj hver angivet URL som startadresse.
-Sitemap address:
-Sitemap-adresse:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adressen på et sitemap, der skal læses i stedet for at undersøge webstedet; lad feltet stå tomt for at undersøge robots.txt og derefter /sitemap.xml.

lang/Deutsch.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Größte eingebettete Ressource (Bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Eine Ressource über dieser Größe behält einen gewöhnlichen Link; leer lassen für den Standardwert von 10485760 Bytes.
-Seed the crawl from the site's sitemap
-Erfassung mit der Sitemap der Website beginnen
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Die Sitemap der Website lesen (Sitemap:-Zeilen in robots.txt, dann /sitemap.xml) und jede dort aufgeführte URL als Startadresse hinzufügen.
-Sitemap address:
-Sitemap-Adresse:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adresse einer Sitemap, die anstelle der Suche auf der Website gelesen wird; leer lassen, um robots.txt und dann /sitemap.xml zu prüfen.

lang/Eesti.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Suurim manustatud ressurss (baiti):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Sellest suurem ressurss säilitab tavalise lingi; jäta tühjaks vaikeväärtuse 10485760 baiti jaoks.
-Seed the crawl from the site's sitemap
-Alusta kogumist saidi saidikaardist
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Loe saidi saidikaarti (robots.txt-i Sitemap:-read, seejärel /sitemap.xml) ja lisa iga seal loetletud URL alguslingina.
-Sitemap address:
-Saidikaardi aadress:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Saidikaardi aadress, mida lugeda saidi sondeerimise asemel; jäta tühjaks, et kontrollida robots.txt-i ja seejärel /sitemap.xml-i.

lang/English.txt

@@ -1024,11 +1024,3 @@ Largest inlined asset (bytes):
 Largest inlined asset (bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
-Seed the crawl from the site's sitemap
-Seed the crawl from the site's sitemap
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Sitemap address:
-Sitemap address:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.

lang/Finnish.txt

@@ -978,11 +978,3 @@ Largest inlined asset (bytes):
 Suurin upotettu resurssi (tavua):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Tätä suurempi resurssi säilyttää tavallisen linkin; jätä tyhjäksi, jolloin käytetään oletusarvoa 10485760 tavua.
-Seed the crawl from the site's sitemap
-Aloita haku sivuston sivukartasta
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Lue sivuston sivukartta (robots.txt-tiedoston Sitemap:-rivit, sitten /sitemap.xml) ja lisää jokainen siinä lueteltu URL-osoite aloitusosoitteeksi.
-Sitemap address:
-Sivukartan osoite:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Luettavan sivukartan osoite sivuston luotaamisen sijaan; jätä tyhjäksi, jolloin tarkistetaan robots.txt ja sitten /sitemap.xml.

lang/Francais.txt

@@ -1024,11 +1024,3 @@ Largest inlined asset (bytes):
 Taille maximale d'une ressource intégrée (octets) :
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Au-delà de cette taille, la ressource reste un lien ordinaire ; laissez vide pour la valeur par défaut de 10485760 octets.
-Seed the crawl from the site's sitemap
-Partir du plan de site (sitemap)
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Lire le plan de site (lignes Sitemap: de robots.txt, puis /sitemap.xml) et ajouter chaque URL listée comme adresse de départ.
-Sitemap address:
-Adresse du plan de site :
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adresse d'un plan de site à lire au lieu de sonder le site ; laissez vide pour sonder robots.txt puis /sitemap.xml.

lang/Greek.txt

@@ -978,11 +978,3 @@ Largest inlined asset (bytes):
 Μέγιστος ενσωματωμένος πόρος (byte):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Πόρος μεγαλύτερος από αυτό το μέγεθος διατηρεί κανονικό σύνδεσμο. Αφήστε το κενό για την προεπιλογή των 10485760 byte.
-Seed the crawl from the site's sitemap
-Έναρξη της ανίχνευσης από τον χάρτη του ιστότοπου
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Ανάγνωση του χάρτη του ιστότοπου (γραμμές Sitemap: στο robots.txt, έπειτα /sitemap.xml) και προσθήκη κάθε διεύθυνσης URL που περιέχει ως αρχικής διεύθυνσης.
-Sitemap address:
-Διεύθυνση χάρτη ιστότοπου:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Διεύθυνση χάρτη ιστότοπου προς ανάγνωση αντί για αναζήτηση στον ιστότοπο. Αφήστε το κενό για έλεγχο του robots.txt και έπειτα του /sitemap.xml.

lang/Italiano.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Dimensione massima della risorsa incorporata (byte):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Una risorsa oltre questa dimensione mantiene un collegamento normale; lasciare vuoto per il valore predefinito di 10485760 byte.
-Seed the crawl from the site's sitemap
-Avvia la scansione dalla mappa del sito
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Legge la mappa del sito (righe Sitemap: in robots.txt, poi /sitemap.xml) e aggiunge come indirizzo iniziale ogni URL elencato.
-Sitemap address:
-Indirizzo della mappa del sito:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Indirizzo di una mappa del sito da leggere invece di sondare il sito; lasciare vuoto per sondare robots.txt e poi /sitemap.xml.

lang/Japanese.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 埋め込む最大サイズ (バイト):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 このサイズを超えるリソースは通常のリンクのままになります。空欄にすると既定値の 10485760 バイトになります。
-Seed the crawl from the site's sitemap
-サイトマップからミラーリングを開始する
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-サイトマップ (robots.txt の Sitemap: 行、次に /sitemap.xml) を読み込み、記載されているすべての URL を開始アドレスとして追加します。
-Sitemap address:
-サイトマップのアドレス:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-サイトを探索する代わりに読み込むサイトマップのアドレス。空欄にすると robots.txt、次に /sitemap.xml を探索します。

lang/Macedonian.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 ½ÐøÓÞÛÕÜ ÒÓàÐÔÕÝ àÕáãàá (ÑÐøâØ):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 ÀÕáãàá ßÞÓÞÛÕÜ ÞÔ ÞÒÐÐ ÓÞÛÕÜØÝÐ ×ÐÔàÖãÒÐ ÞÑØçÝÐ ÒàáÚÐ; ÞáâÐÒÕâÕ ßàÐ×ÝÞ ×Ð áâÐÝÔÐàÔÝØâÕ 10485760 ÑÐøâØ.
-Seed the crawl from the site's sitemap
-·ÐßÞçÝØ ÓÞ ßàÕÑÐàãÒÐúÕâÞ ÞÔ ÚÐàâÐâÐ ÝÐ áÐøâÞâ
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-¿àÞçØâÐø øÐ ÚÐàâÐâÐ ÝÐ áÐøâÞâ (àÕÔÞÒØâÕ Sitemap: ÒÞ robots.txt, ßÞâÞÐ /sitemap.xml) Ø ÔÞÔÐø øÐ áÕÚÞøÐ ÝÐÒÕÔÕÝÐ URL ÐÔàÕáÐ ÚÐÚÞ ßÞçÕâÝÐ.
-Sitemap address:
-°ÔàÕáÐ ÝÐ ÚÐàâÐâÐ ÝÐ áÐøâÞâ:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-°ÔàÕáÐ ÝÐ ÚÐàâÐ ÝÐ áÐøâÞâ èâÞ âàÕÑÐ ÔÐ áÕ ßàÞçØâÐ ÝÐÜÕáâÞ ØáßØâãÒÐúÕ ÝÐ áÐøâÞâ; ÞáâÐÒÕâÕ ßàÐ×ÝÞ ×Ð ÔÐ áÕ ØáßØâÐ robots.txt, ßÐ /sitemap.xml.

lang/Magyar.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Legnagyobb beágyazott erõforrás (bájt):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Az ennél nagyobb erõforrás közönséges hivatkozás marad; hagyja üresen a 10485760 bájtos alapértelmezéshez.
-Seed the crawl from the site's sitemap
-A letöltés indítása a webhely webhelytérképérõl
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-A webhely webhelytérképének beolvasása (a robots.txt Sitemap: sorai, majd a /sitemap.xml), és a benne felsorolt összes URL felvétele kiindulási címként.
-Sitemap address:
-Webhelytérkép címe:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-A webhely vizsgálata helyett beolvasandó webhelytérkép címe; hagyja üresen a robots.txt, majd a /sitemap.xml vizsgálatához.

lang/Nederlands.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Grootste ingesloten bron (bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Een bron boven deze grootte houdt een gewone koppeling; laat leeg voor de standaardwaarde van 10485760 bytes.
-Seed the crawl from the site's sitemap
-De crawl starten vanaf de sitemap van de site
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-De sitemap van de site lezen (Sitemap:-regels in robots.txt, daarna /sitemap.xml) en elke vermelde URL als startadres toevoegen.
-Sitemap address:
-Sitemap-adres:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adres van een sitemap die gelezen moet worden in plaats van de site te onderzoeken; laat leeg om robots.txt en daarna /sitemap.xml te controleren.

lang/Norsk.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Største innebygde ressurs (byte):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 En ressurs over denne størrelsen beholder en vanlig lenke; la feltet stå tomt for standardverdien på 10485760 byte.
-Seed the crawl from the site's sitemap
-Start gjennomgangen fra nettstedets nettstedskart
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Les nettstedets nettstedskart (Sitemap:-linjer i robots.txt, deretter /sitemap.xml) og legg til hver oppført URL som startadresse.
-Sitemap address:
-Adresse til nettstedskart:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adressen til et nettstedskart som skal leses i stedet for å undersøke nettstedet; la feltet stå tomt for å undersøke robots.txt og deretter /sitemap.xml.

lang/Polski.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Najwiêkszy osadzony zasób (bajty):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Zasób wiêkszy ni¿ ten rozmiar zachowuje zwyk³y odno¶nik; pozostaw puste, aby u¿yæ domy¶lnych 10485760 bajtów.
-Seed the crawl from the site's sitemap
-Rozpocznij pobieranie od mapy witryny
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Odczytaj mapê witryny (wiersze Sitemap: w pliku robots.txt, nastêpnie /sitemap.xml) i dodaj ka¿dy wymieniony adres URL jako adres pocz±tkowy.
-Sitemap address:
-Adres mapy witryny:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adres mapy witryny do odczytania zamiast sondowania witryny; pozostaw puste, aby sprawdziæ robots.txt, a nastêpnie /sitemap.xml.

lang/Portugues-Brasil.txt

@@ -1024,11 +1024,3 @@ Largest inlined asset (bytes):
 Maior recurso incorporado (bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Um recurso acima desse tamanho mantém um link comum; deixe em branco para o padrão de 10485760 bytes.
-Seed the crawl from the site's sitemap
-Iniciar a captura pelo mapa do site
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Ler o mapa do site (linhas Sitemap: do robots.txt, depois /sitemap.xml) e adicionar como endereço inicial cada URL nele listada.
-Sitemap address:
-Endereço do mapa do site:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Endereço de um mapa do site a ser lido em vez de sondar o site; deixe em branco para sondar robots.txt e depois /sitemap.xml.

lang/Portugues.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Maior recurso incorporado (bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Um recurso acima deste tamanho mantém uma ligação normal; deixe em branco para o valor predefinido de 10485760 bytes.
-Seed the crawl from the site's sitemap
-Iniciar a recolha pelo mapa do site
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Ler o mapa do site (linhas Sitemap: do robots.txt, depois /sitemap.xml) e adicionar como endereço inicial cada URL nele listado.
-Sitemap address:
-Endereço do mapa do site:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Endereço de um mapa do site a ler em vez de sondar o site; deixe em branco para sondar robots.txt e depois /sitemap.xml.

lang/Romanian.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Cea mai mare resursa încorporata (octeti):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 O resursa mai mare decât aceasta dimensiune pastreaza o legatura obisnuita; lasati gol pentru valoarea implicita de 10485760 de octeti.
-Seed the crawl from the site's sitemap
-Porneste explorarea de la harta sitului
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Citeste harta sitului (liniile Sitemap: din robots.txt, apoi /sitemap.xml) si adauga fiecare URL listat ca adresa de pornire.
-Sitemap address:
-Adresa hartii sitului:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adresa unei harti a sitului care sa fie citita în loc de sondarea sitului; lasati gol pentru a sonda robots.txt, apoi /sitemap.xml.

lang/Russian.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Íàèáîëüøèé âñòðàèâàåìûé ðåñóðñ (áàéòû):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Ðåñóðñ áîëüøå ýòîãî ðàçìåðà ñîõðàíÿåò îáû÷íóþ ññûëêó; îñòàâüòå ïóñòûì äëÿ çíà÷åíèÿ ïî óìîë÷àíèþ 10485760 áàéò.
-Seed the crawl from the site's sitemap
-Íà÷èíàòü îáõîä ñ êàðòû ñàéòà
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Ïðî÷èòàòü êàðòó ñàéòà (ñòðîêè Sitemap: â robots.txt, çàòåì /sitemap.xml) è äîáàâèòü êàæäûé óêàçàííûé â íåé URL êàê íà÷àëüíûé àäðåñ.
-Sitemap address:
-Àäðåñ êàðòû ñàéòà:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Àäðåñ êàðòû ñàéòà, êîòîðóþ íóæíî ïðî÷èòàòü âìåñòî îïðîñà ñàéòà; îñòàâüòå ïóñòûì, ÷òîáû ïðîâåðèòü robots.txt, çàòåì /sitemap.xml.

lang/Slovak.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Najväè¹í vlo¾ený zdroj (bajty):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Zdroj väè¹í ne¾ táto veµkos» si ponechá be¾ný odkaz; ponechajte prázdne pre predvolených 10485760 bajtov.
-Seed the crawl from the site's sitemap
-Zaèa» prehliadanie z mapy stránok
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Naèíta» mapu stránok (riadky Sitemap: v súbore robots.txt, potom /sitemap.xml) a prida» ka¾dú uvedenú adresu URL ako poèiatoènú.
-Sitemap address:
-Adresa mapy stránok:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adresa mapy stránok, ktorá sa má naèíta» namiesto zis»ovania na stránke; ponechajte prázdne na zistenie z robots.txt a potom /sitemap.xml.

lang/Slovenian.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Najvecji vgrajeni vir (bajti):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Vir, vecji od te velikosti, ohrani obicajno povezavo; pustite prazno za privzetih 10485760 bajtov.
-Seed the crawl from the site's sitemap
-Zacni zajem z zemljevidom spletnega mesta
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Preberi zemljevid spletnega mesta (vrstice Sitemap: v robots.txt, nato /sitemap.xml) in dodaj vsak navedeni URL kot zacetni naslov.
-Sitemap address:
-Naslov zemljevida spletnega mesta:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Naslov zemljevida spletnega mesta, ki naj se prebere namesto preverjanja mesta; pustite prazno za preverjanje robots.txt in nato /sitemap.xml.

lang/Svenska.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Största inbäddade resurs (byte):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 En resurs över den här storleken behåller en vanlig länk; lämna tomt för standardvärdet 10485760 byte.
-Seed the crawl from the site's sitemap
-Starta insamlingen från webbplatsens webbplatskarta
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Läs webbplatsens webbplatskarta (Sitemap:-rader i robots.txt, sedan /sitemap.xml) och lägg till varje angiven URL som startadress.
-Sitemap address:
-Webbplatskartans adress:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Adress till en webbplatskarta som ska läsas i stället för att söka på webbplatsen; lämna tomt för att kontrollera robots.txt och sedan /sitemap.xml.

lang/Turkish.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 En büyük gömülü kaynak (bayt):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Bu boyutun üzerindeki bir kaynak sýradan baðlantýsýný korur; 10485760 baytlýk varsayýlan için boþ býrakýn.
-Seed the crawl from the site's sitemap
-Taramayý sitenin site haritasýndan baþlat
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Sitenin site haritasýný oku (robots.txt içindeki Sitemap: satýrlarý, ardýndan /sitemap.xml) ve listelenen her URL'yi baþlangýç adresi olarak ekle.
-Sitemap address:
-Site haritasý adresi:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Siteyi yoklamak yerine okunacak site haritasýnýn adresi; robots.txt ve ardýndan /sitemap.xml yoklamasý için boþ býrakýn.

lang/Ukrainian.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Íàéá³ëüøèé âáóäîâàíèé ðåñóðñ (áàéòè):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Ðåñóðñ, á³ëüøèé çà öåé ðîçì³ð, çáåð³ãຠçâè÷àéíå ïîñèëàííÿ; çàëèøòå ïîðîæí³ì äëÿ òèïîâîãî çíà÷åííÿ 10485760 áàéò³â.
-Seed the crawl from the site's sitemap
-Ïî÷èíàòè îáõ³ä ç êàðòè ñàéòó
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Ïðî÷èòàòè êàðòó ñàéòó (ðÿäêè Sitemap: ó robots.txt, ïîò³ì /sitemap.xml) ³ äîäàòè êîæíó âêàçàíó â í³é URL-àäðåñó ÿê ïî÷àòêîâó.
-Sitemap address:
-Àäðåñà êàðòè ñàéòó:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Àäðåñà êàðòè ñàéòó, ÿêó ñë³ä ïðî÷èòàòè çàì³ñòü îïèòóâàííÿ ñàéòó; çàëèøòå ïîðîæí³ì, ùîá ïåðåâ³ðèòè robots.txt, à ïîò³ì /sitemap.xml.

lang/Uzbek.txt

@@ -976,11 +976,3 @@ Largest inlined asset (bytes):
 Eng katta joylangan resurs (bayt):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Bu o’lchamdan katta resurs oddiy havolani saqlab qoladi; standart 10485760 bayt uchun bo’sh qoldiring.
-Seed the crawl from the site's sitemap
-Yig’ishni saytning sayt xaritasidan boshlash
-Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
-Saytning sayt xaritasini o’qish (robots.txt dagi Sitemap: qatorlari, so’ngra /sitemap.xml) va unda ko’rsatilgan har bir URL manzilni boshlang’ich manzil sifatida qo’shish.
-Sitemap address:
-Sayt xaritasi manzili:
-Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
-Saytni tekshirish o’rniga o’qiladigan sayt xaritasi manzili; robots.txt, so’ngra /sitemap.xml ni tekshirish uchun bo’sh qoldiring.

m4/check_zlib.m4

@@ -1,33 +1,82 @@
 dnl @synopsis CHECK_ZLIB()
 dnl
-dnl Look for zlib. It is a hard requirement, not an option: the cache and the
-dnl WARC output are zip/gzip containers, and the bundled minizip calls zlib
-dnl directly. --with-zlib=DIR points at a non-standard prefix.
+dnl This macro searches for an installed zlib library. If nothing
+dnl was specified when calling configure, it searches first in /usr/local
+dnl and then in /usr. If the --with-zlib=DIR is specified, it will try
+dnl to find it in DIR/include/zlib.h and DIR/lib/libz.a. If --without-zlib
+dnl is specified, the library is not searched at all.
+dnl
+dnl If either the header file (zlib.h) or the library (libz) is not
+dnl found, the configuration exits on error, asking for a valid
+dnl zlib installation directory or --without-zlib.
+dnl
+dnl The macro defines the symbol HAVE_LIBZ if the library is found. You should
+dnl use autoheader to include a definition for this symbol in a config.h
+dnl file. Sample usage in a C/C++ source is as follows:
+dnl
+dnl   #ifdef HAVE_LIBZ
+dnl   #include <zlib.h>
+dnl   #endif /* HAVE_LIBZ */
+dnl
+dnl @version $Id$
+dnl @author Loic Dachary <loic@senga.org>
 dnl
-dnl Adds -lz to LIBS and defines HAVE_LIBZ.
 
-AC_DEFUN([CHECK_ZLIB], [
-AC_ARG_WITH([zlib],
-	[AS_HELP_STRING([--with-zlib=DIR],[root directory of the zlib installation])],
-	[zlib_want=$withval], [zlib_want=yes])
-if test "$zlib_want" = "no"; then
-	AC_MSG_ERROR([zlib cannot be disabled: the cache and the WARC output are zip/gzip containers, and the bundled minizip calls zlib directly])
+AC_DEFUN([CHECK_ZLIB],
+#
+# Handle user hints
+#
+[AC_MSG_CHECKING(if zlib is wanted)
+AC_ARG_WITH(zlib,
+[  --with-zlib=DIR root directory path of zlib installation [defaults to
+                    /usr/local or /usr if not found in /usr/local]
+  --without-zlib to disable zlib usage completely],
+[if test "$withval" != no ; then
+  AC_MSG_RESULT(yes)
+  ZLIB_HOME="$withval"
+else
+  AC_MSG_RESULT(no)
+fi], [
+AC_MSG_RESULT(yes)
+ZLIB_HOME=/usr/local
+if test ! -f "${ZLIB_HOME}/include/zlib.h"
+then
+        ZLIB_HOME=/usr
 fi
-if test "$zlib_want" != "yes"; then
-	# An explicit prefix is authoritative: if the header is not under it,
-	# error rather than silently pick a system copy.
-	if test ! -f "$zlib_want/include/zlib.h"; then
-		AC_MSG_ERROR([zlib requested at $zlib_want but $zlib_want/include/zlib.h is missing])
-	fi
-	CPPFLAGS="$CPPFLAGS -I$zlib_want/include"
-	LDFLAGS="$LDFLAGS -L$zlib_want/lib"
-elif test -f /usr/local/include/zlib.h; then
-	# Where the BSD ports tree lands zlib, and not always searched by default.
-	CPPFLAGS="$CPPFLAGS -I/usr/local/include"
-	LDFLAGS="$LDFLAGS -L/usr/local/lib"
-fi
-AC_CHECK_HEADER([zlib.h], [],
-	[AC_MSG_ERROR([zlib.h not found; install the zlib development files or pass --with-zlib=DIR])])
-AC_CHECK_LIB([z], [inflateEnd], [],
-	[AC_MSG_ERROR([libz not found; install the zlib development files or pass --with-zlib=DIR])])
+])
+
+#
+# Locate zlib, if wanted
+#
+if test -n "${ZLIB_HOME}"
+then
+        ZLIB_OLD_LDFLAGS=$LDFLAGS
+        ZLIB_OLD_CPPFLAGS=$LDFLAGS
+        LDFLAGS="$LDFLAGS -L${ZLIB_HOME}/lib"
+        CPPFLAGS="$CPPFLAGS -I${ZLIB_HOME}/include"
+        AC_LANG_SAVE
+        AC_LANG_C
+        AC_CHECK_LIB(z, inflateEnd, [zlib_cv_libz=yes], [zlib_cv_libz=no])
+        AC_CHECK_HEADER(zlib.h, [zlib_cv_zlib_h=yes], [zlib_cv_zlib_h=no])
+        AC_LANG_RESTORE
+        if test "$zlib_cv_libz" = "yes" -a "$zlib_cv_zlib_h" = "yes"
+        then
+                #
+                # If both library and header were found, use them
+                #
+                AC_CHECK_LIB(z, inflateEnd)
+                AC_MSG_CHECKING(zlib in ${ZLIB_HOME})
+                AC_MSG_RESULT(ok)
+        else
+                #
+                # If either header or library was not found, revert and bomb
+                #
+                AC_MSG_CHECKING(zlib in ${ZLIB_HOME})
+                LDFLAGS="$ZLIB_OLD_LDFLAGS"
+                CPPFLAGS="$ZLIB_OLD_CPPFLAGS"
+                AC_MSG_RESULT(failed)
+                AC_MSG_ERROR(either specify a valid zlib installation with --with-zlib=DIR or disable zlib usage with --without-zlib)
+        fi
+fi
+
 ])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "27 July 2026" "httrack website copier"
+.TH httrack 1 "26 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -36,7 +36,6 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-t, \-\-test\fR ]
 [ \fB\-%L, \-\-list\fR ]
 [ \fB\-%S, \-\-urllist\fR ]
-[ \fB\-%m, \-\-sitemap\fR ]
 [ \fB\-NN, \-\-structure[=N]\fR ]
 [ \fB\-%N, \-\-delayed\-type\-check\fR ]
 [ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
@@ -190,8 +189,6 @@ test all URLs (even forbidden ones) (\-\-test)
 <file> add all URL located in this text file (one URL per line) (\-\-list <param>)
 .IP \-%S
 <file> add all scan rules located in this text file (one scan rule per line) (\-\-urllist <param>)
-.IP \-%m
-seed the crawl from the site's sitemap (robots.txt Sitemap:, then /sitemap.xml); \-\-sitemap\-url URL names one explicitly. A sitemap you name, or one the site declares, is fetched even under robots.txt Disallow; only the guessed /sitemap.xml obeys it. The URLs found still pass every filter and scope rule (\-\-sitemap)
 .SS Build options:
 .IP \-NN
 structure type (0 *original structure, 1+: see below) (\-\-structure[=N])
@@ -220,7 +217,7 @@ do not include any password for external password protected websites (%x0 includ
 .IP \-%g
 strip query keys for dedup ([host/pattern=]key1,key2,...) (\-\-strip\-query <param>)
 .IP \-o
-*save the server's error pages (404..) (o0 discard them) (\-\-generate\-errors)
+*generate output html file in case of error (404..) (o0 don't generate) (\-\-generate\-errors)
 .IP \-X
 *purge old files after update (X0 keep delete) (\-\-purge\-old[=N])
 .IP \-%p

src/Makefile.am

@@ -66,7 +66,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
-	htsmd5.c htscodec.c htswarc.c htschanges.c htssinglefile.c htssitemap.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
+	htsmd5.c htscodec.c htswarc.c htschanges.c htssinglefile.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
 	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
@@ -77,7 +77,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
 	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
-	htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htschanges.h htssinglefile.h htssitemap.h htsproxy.h htszlib.h  \
+	htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htschanges.h htssinglefile.h htsproxy.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \
 	htscharset.h punycode.h htsencoding.h \
 	htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \

src/hts-indextmpl.h

@@ -487,6 +487,141 @@ regen:
 
 #define HTS_DATA_UNKNOWN_HTML_LEN 0
 
+#define HTS_DATA_ERROR_HTML "<html>"LF\
+  "<html xmlns=\"http://www.w3.org/1999/xhtml\" lang=\"en\">"LF\
+  ""LF\
+  "<head>"LF\
+  "	<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />"LF\
+  "	<meta name=\"description\" content=\"HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help\" />"LF\
+  "	<meta name=\"keywords\" content=\"httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local  site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software\" />"LF\
+  "	<title>Page not retrieved! - HTTrack Website Copier</title>"LF\
+  "	<style type=\"text/css\">"LF\
+  "	<!--"LF\
+  ""LF\
+  "body {"LF\
+  "	margin: 0;  padding: 0;  margin-bottom: 15px;  margin-top: 8px;"LF\
+  "	background: #77b;"LF\
+  "}"LF\
+  "body, td {"LF\
+  "	font: 14px \"Trebuchet MS\", Verdana, Arial, Helvetica, sans-serif;"LF\
+  "	}"LF\
+  ""LF\
+  "#subTitle {"LF\
+  "	background: #000;  color: #fff;  padding: 4px;  font-weight: bold; "LF\
+  "	}"LF\
+  ""LF\
+  "#siteNavigation a, #siteNavigation .current {"LF\
+  "	font-weight: bold;  color: #448;"LF\
+  "	}"LF\
+  "#siteNavigation a:link    { text-decoration: none; }"LF\
+  "#siteNavigation a:visited { text-decoration: none; }"LF\
+  ""LF\
+  "#siteNavigation .current { background-color: #ccd; }"LF\
+  ""LF\
+  "#siteNavigation a:hover   { text-decoration: none;  background-color: #fff;  color: #000; }"LF\
+  "#siteNavigation a:active  { text-decoration: none;  background-color: #ccc; }"LF\
+  ""LF\
+  ""LF\
+  "a:link    { text-decoration: underline;  color: #00f; }"LF\
+  "a:visited { text-decoration: underline;  color: #000; }"LF\
+  "a:hover   { text-decoration: underline;  color: #c00; }"LF\
+  "a:active  { text-decoration: underline; }"LF\
+  ""LF\
+  "#pageContent {"LF\
+  "	clear: both;"LF\
+  "	border-bottom: 6px solid #000;"LF\
+  "	padding: 10px;  padding-top: 20px;"LF\
+  "	line-height: 1.65em;"LF\
+	" background-image: url(backblue.gif);"LF\
+	" background-repeat: no-repeat;"LF\
+	" background-position: top right;"LF\
+  "	}"LF\
+  ""LF\
+  "#pageContent, #siteNavigation {"LF\
+  "	background-color: #ccd;"LF\
+  "	}"LF\
+  ""LF\
+  ""LF\
+  ".imgLeft  { float: left;   margin-right: 10px;  margin-bottom: 10px; }"LF\
+  ".imgRight { float: right;  margin-left: 10px;   margin-bottom: 10px; }"LF\
+  ""LF\
+  "hr { height: 1px;  color: #000;  background-color: #000;  margin-bottom: 15px; }"LF\
+  ""LF\
+  "h1 { margin: 0;  font-weight: bold;  font-size: 2em; }"LF\
+  "h2 { margin: 0;  font-weight: bold;  font-size: 1.6em; }"LF\
+  "h3 { margin: 0;  font-weight: bold;  font-size: 1.3em; }"LF\
+  "h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }"LF\
+  ""LF\
+  ".blak { background-color: #000; }"LF\
+  ".hide { display: none; }"LF\
+  ".tableWidth { min-width: 400px; }"LF\
+  ""LF\
+  ".tblRegular       { border-collapse: collapse; }"LF\
+  ".tblRegular td    { padding: 6px;  background-image: url(fade.gif);  border: 2px solid #99c; }"LF\
+  ".tblHeaderColor, .tblHeaderColor td { background: #99c; }"LF\
+  ".tblNoBorder td   { border: 0; }"LF\
+  ""LF\
+  ""LF\
+  "// -->"LF\
+  "</style>"LF\
+  ""LF\
+  "</head>"LF\
+  ""LF\
+  "<table width=\"76%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"3\" class=\"tableWidth\">"LF\
+  "	<tr>"LF\
+  "	<td id=\"subTitle\">HTTrack Website Copier - Open Source offline browser</td>"LF\
+  "	</tr>"LF\
+  "</table>"LF\
+  "<table width=\"76%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"0\" class=\"tableWidth\">"LF\
+  "<tr class=\"blak\">"LF\
+  "<td>"LF\
+  "	<table width=\"100%%\" border=\"0\" align=\"center\" cellspacing=\"1\" cellpadding=\"0\">"LF\
+  "	<tr>"LF\
+  "	<td colspan=\"6\"> "LF\
+  "		<table width=\"100%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"10\">"LF\
+  "		<tr> "LF\
+  "		<td id=\"pageContent\"> "LF\
+  "<!-- ==================== End prologue ==================== -->"LF\
+  "<h1><strong><u>Oops!...</u></strong></h1>"LF\
+  "<h3>This page has <font color=\"red\"><em>not</em></font> been retrieved by HTTrack Website Copier (%s). </h3>"LF\
+  "<script language=\"Javascript\">"LF\
+  "<!--"LF\
+  "  var loc=document.location.toString();"LF\
+  "  if (loc) {"LF\
+  "    var pos=loc.indexOf('link=');"LF\
+  "    if (pos>0) {"LF\
+  "      document.write('Clic to the link <b>below</b> to go to the online location!<br><a href=\"'+loc.substring(pos+5)+'\">'+loc.substring(pos+5)+'</a><br>');"LF\
+  "    } else"LF\
+  "      document.write('(no location defined)');"LF\
+  "  }"LF\
+  "// -->"LF\
+  "</script>"LF\
+  "<h6 align=\"right\">Mirror by HTTrack Website Copier</h6>"LF\
+  "</body>"LF\
+  "</html>"LF\
+  "<!-- ==================== Start epilogue ==================== -->"LF\
+  "		</td>"LF\
+  "		</tr>"LF\
+  "		</table>"LF\
+  "	</td>"LF\
+  "	</tr>"LF\
+  "	</table>"LF\
+  "</td>"LF\
+  "</tr>"LF\
+  "</table>"LF\
+  ""LF\
+  "<table width=\"76%%\" height=\"100%%\" border=\"0\" align=\"center\" valign=\"bottom\" cellspacing=\"0\" cellpadding=\"0\">"LF\
+  "	<tr>"LF\
+  "	<td id=\"footer\"><small>&copy; 2014 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\
+  "	</tr>"LF\
+  "</table>"LF\
+  ""LF\
+  "</body>"LF\
+  ""LF\
+  "</html>"LF\
+  ""LF\
+  ""LF
+
 // image gif "unknown"
 #define HTS_DATA_UNKNOWN_GIF \
   "\x47\x49\x46\x38\x39\x61\x20\x0\x20\x0\xf7\xff\x0\xc0\xc0\xc0\xff\x0\x0\xfc\x3\x0\xf8\x6\x0\xf6\x9\x0\xf2\xc\x0\xf0\xf\x0\xf0\xe\x0\xed\x11\x0\xec\x13\x0\xeb\x14\x0\xe9\x15\x0\xe8\x18\x0\xe6\x18\x0\xe5\x1a\x0\xe3\x1c\x0\xe2\x1d\x0\xe1\x1e\x0\xdf\x20\x0\xdd\x23\x0\xdd\x22\x0\xdb\x23\x0\xda\x25\x0\xd9\x25\x0\xd8\x27\x0\xd6\x29\x0\xd5\x2a\x0\xd3\x2c\x0\xd2\x2d\x0"\

src/htsalias.c

@@ -116,10 +116,6 @@ const char *hts_optalias[][4] = {
    "load extra cookies from a Netscape cookies.txt"},
   {"changes", "-%d", "single",
    "write hts-changes.json: what this crawl changed vs. the previous mirror"},
-  {"sitemap", "-%m", "single",
-   "seed the crawl from the start host's sitemap (robots.txt, then "
-   "/sitemap.xml)"},
-  {"sitemap-url", "-%mu", "param1", "seed the crawl from this sitemap URL"},
   {"warc", "-%r", "single", "write an ISO-28500 WARC/1.1 archive of the crawl"},
   {"warc-file", "-%rf", "param1", "write a WARC archive to the given base name"},
   {"warc-max-size", "-%rs", "param1",
@@ -501,11 +497,13 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
       /* read line */
       linput(fp, line, 250);
       hts_lowcase(line);
-      /* trim first: a blank line is skipped, not parsed as an option */
-      hts_rtrim(line, HTS_REALSPACES);
       if (strnotempty(line)) {
         /* no comment line: # // ; */
         if (strchr("#/;", line[0]) == NULL) {
+          /* right trim */
+          a = line + strlen(line) - 1;
+          while(is_realspace(*a))
+            *(a--) = '\0';
           /* jump "set " and spaces */
           a = line;
           while(is_realspace(*a))
@@ -575,6 +573,7 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
             }
           }
         }
+
       }
     }
     fclose(fp);

src/htsback.c

@@ -48,6 +48,11 @@ Please visit our Website: http://www.httrack.com
 #include "htsftp.h"
 #include "htscodec.h"
 #include "htsproxy.h"
+#if HTS_USEZLIB
+#include "htszlib.h"
+#else
+#error HTS_USEZLIB not defined
+#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus
@@ -340,111 +345,12 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr
 }
 
 static int slot_can_be_cached_on_disk(const lien_back * back) {
-  /* A pending backup or spool means the slot is not finalized, and the swap
-     would unlink it through back_clear_entry() (#771). */
-  if (back->tmpfile != NULL && back->tmpfile[0] != '\0')
-    return 0;
   return (back->status == STATUS_READY && back->locked == 0
           && back->url_sav[0] != '\0'
           && strcmp(back->url_sav, BACK_ADD_TEST) != 0);
   /* Note: not checking !IS_DELAYED_EXT(back->url_sav) or it will quickly cause the slots to be filled! */
 }
 
-int back_selftest_slot_swap(void) {
-  lien_back back;
-  int err = 0;
-
-#define CHECK(want, why)                                                       \
-  do {                                                                         \
-    if (slot_can_be_cached_on_disk(&back) != (want)) {                         \
-      fprintf(stderr, "backswap: expected %d for %s\n", (want), (why));        \
-      err = 1;                                                                 \
-    }                                                                          \
-  } while (0)
-
-  memset(&back, 0, sizeof(back));
-  back.status = STATUS_READY;
-  strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
-  CHECK(1, "a plain ready slot");
-
-  back.tmpfile = back.tmpfile_buffer;
-  strcpybuff(back.tmpfile_buffer, "/tmp/httrack-selftest.bin.bak");
-  CHECK(0, "a slot still holding a re-fetch backup");
-
-  /* Callers clear a spent temporary by emptying the name, not the pointer. */
-  back.tmpfile_buffer[0] = '\0';
-  CHECK(1, "a slot whose temporary was already dropped");
-
-  back.tmpfile = NULL;
-  back.locked = 1;
-  CHECK(0, "a locked slot");
-  back.locked = 0;
-
-  back.status = STATUS_TRANSFER;
-  CHECK(0, "a slot still transferring");
-  back.status = STATUS_READY;
-
-  back.url_sav[0] = '\0';
-  CHECK(0, "a slot with no save name");
-
-  strcpybuff(back.url_sav, BACK_ADD_TEST);
-  CHECK(0, "the dummy test slot");
-#undef CHECK
-
-  /* The swap round-trip must not lose the size of a slot whose body is already
-     at url_sav, or the link writer blanks the file (#797). */
-  {
-    static const char body[] = "swapped body";
-    int c;
-
-    for (c = 0; c < 2; c++) {
-      const hts_boolean inmemory = c == 0 ? HTS_TRUE : HTS_FALSE;
-      FILE *const fp = tmpfile();
-      lien_back *copy = NULL;
-
-      memset(&back, 0, sizeof(back));
-      back.status = STATUS_READY;
-      strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
-      back.r.size = (LLint) sizeof(body) - 1;
-      if (inmemory) {
-        back.r.adr = strdupt(body);
-      }
-      if (fp == NULL || back_serialize(fp, &back) != 0 ||
-          fseek(fp, 0, SEEK_SET) != 0 || back_unserialize(fp, &copy) != 0) {
-        fprintf(stderr, "backswap: round-trip failed for a %s slot\n",
-                inmemory ? "buffered" : "direct-to-disk");
-        err = 1;
-      } else {
-        if (copy->r.size != back.r.size) {
-          fprintf(stderr,
-                  "backswap: %s slot came back with size " LLintP
-                  ", expected " LLintP "\n",
-                  inmemory ? "buffered" : "direct-to-disk", copy->r.size,
-                  back.r.size);
-          err = 1;
-        }
-        if (inmemory && (copy->r.adr == NULL ||
-                         memcmp(copy->r.adr, body, sizeof(body) - 1) != 0)) {
-          fprintf(stderr, "backswap: buffered slot lost its body\n");
-          err = 1;
-        }
-        if (!inmemory && copy->r.adr != NULL) {
-          fprintf(stderr, "backswap: direct-to-disk slot gained a body\n");
-          err = 1;
-        }
-        back_clear_entry(copy);
-        freet(copy);
-      }
-      if (fp != NULL)
-        fclose(fp);
-      freet(back.r.adr);
-    }
-  }
-
-  printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
 /* Put all backing entries that are ready in the storage hashtable to spare space and CPU */
 int back_cleanup_background(httrackp * opt, cache_back * cache,
                             struct_back * sback) {
@@ -483,27 +389,18 @@ int back_cleanup_background(httrackp * opt, cache_back * cache,
 #ifndef HTS_NO_BACK_ON_DISK
       /* temporarily serialize the entry on disk */
       {
-        /* +16: room for the ".tmp" the url_sav form appends to a full-length
-           save name, so one buffer holds both shapes */
-        char BIGSTK tmpname[HTS_URLMAXSIZE * 2 + 16];
-        char *filename;
-        hts_boolean named;
-
-        /* the -p0 name is not derived from url_sav, so it needs a buffer of
-           its own size rather than the save name's */
-        if (opt->getmode != 0) {
-          named =
-              slprintfbuff(tmpname, sizeof(tmpname), "%s.tmp", back[i].url_sav);
-        } else {
-          named = slprintfbuff(tmpname, sizeof(tmpname), "%stmpfile%d.tmp",
-                               StringBuff(opt->path_html_utf8),
-                               opt->state.tmpnameid++);
-        }
-        filename = named ? strdupt(tmpname) : NULL;
+        int fsz = (int) strlen(back[i].url_sav);
+        char *filename = malloc(fsz + 8 + 1);
 
         if (filename != NULL) {
           FILE *fp;
 
+          if (opt->getmode != 0) {
+            sprintf(filename, "%s.tmp", back[i].url_sav);
+          } else {
+            sprintf(filename, "%stmpfile%d.tmp",
+                    StringBuff(opt->path_html_utf8), opt->state.tmpnameid++);
+          }
           /* Security check */
           if (fexist_utf8(filename)) {
             hts_log_print(opt, LOG_WARNING,
@@ -534,12 +431,11 @@ int back_cleanup_background(httrackp * opt, cache_back * cache,
                           "file does not exist");
           }
           if (filename != NULL)
-            freet(filename);
+            free(filename);
         } else {
           hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
-                        "engine: warning: serialize error for %s%s: %s",
-                        back[i].url_adr, back[i].url_fil,
-                        named ? "memory full" : "temporary filename too long");
+                        "engine: warning: serialize error for %s%s: memory full",
+                        back[i].url_adr, back[i].url_fil);
         }
       }
 #else
@@ -656,42 +552,6 @@ int back_nsoc_overall(const struct_back * sback) {
   return n;
 }
 
-/* Subdirectory holding a mirrored file's temporaries, beside it. url_savename()
-   maps '~' to '_', so no URL can ever be mirrored inside it (#774, #842). */
-#define HTS_TMPDIR "~hts-tmp"
-
-/* Build save's temporary as <dir>/<HTS_TMPDIR>/<name>.<ext>. Appending the
-   extension to save instead put it in the mirror namespace, so a site serving
-   <path>.bak had its copy taken as the backup and then unlinked (#774).
-   HTS_FALSE (dest emptied) if it would not fit. Note: utf-8. */
-static hts_boolean back_tmpname(char *dest, size_t size, const char *save,
-                                const char *ext) {
-  const char *const slash = strrchr(save, '/');
-  const int dirlen = slash != NULL ? (int) (slash - save) + 1 : 0;
-
-  if (!slprintfbuff(dest, size, "%.*s" HTS_TMPDIR "/%s.%s", dirlen, save,
-                    slash != NULL ? slash + 1 : save, ext)) {
-    dest[0] = '\0';
-    return HTS_FALSE;
-  }
-  return HTS_TRUE;
-}
-
-/* Note: utf-8 */
-void back_tmpdir_drop(const char *tmp) {
-  char BIGSTK dir[HTS_URLMAXSIZE * 2];
-  const char *slash;
-
-  if (tmp == NULL || (slash = strrchr(tmp, '/')) == NULL)
-    return;
-  if (!strclipbuff(dir, sizeof(dir), tmp))
-    return;
-  dir[slash - tmp] = '\0';
-  slash = strrchr(dir, '/');
-  if (strcmp(slash != NULL ? slash + 1 : dir, HTS_TMPDIR) == 0)
-    (void) RMDIR(dir);
-}
-
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
 static int create_back_tmpfile(httrackp *opt, lien_back *const back,
@@ -699,25 +559,23 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav[0] != '\0') {
-    if (!back_tmpname(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
-                      back->url_sav, ext)) {
+    /* same capacity as url_sav, so truncation drops the extension and aliases
+       the temp name onto the live file that back_finalize_backup() UNLINKs */
+    if (!sprintfbuff(back->tmpfile_buffer, "%s.%s", back->url_sav, ext)) {
       hts_log_print(opt, LOG_WARNING, "temporary filename too long for %s",
                     back->url_sav);
+      back->tmpfile_buffer[0] = '\0';
       return -1;
     }
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
       hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
                     back->tmpfile);
-      back->tmpfile_buffer[0] = '\0';
-      back->tmpfile = NULL;
       return -1;
     }
   } else {
-    /* same directory as the named case, so back_tmpdir_drop() only removes one
-       the engine made (#842) */
     /* truncation here would collide distinct tmpnameid's onto one name */
-    if (!sprintfbuff(back->tmpfile_buffer, "%s/" HTS_TMPDIR "/tmp%d.%s",
+    if (!sprintfbuff(back->tmpfile_buffer, "%s/tmp%d.%s",
                      StringBuff(opt->path_html_utf8), opt->state.tmpnameid++,
                      ext)) {
       hts_log_print(opt, LOG_WARNING, "temporary filename too long in %s",
@@ -726,75 +584,30 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
       return -1;
     }
     back->tmpfile = back->tmpfile_buffer;
-    if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
-                    back->tmpfile);
-      back->tmpfile_buffer[0] = '\0';
-      back->tmpfile = NULL;
-      return -1;
-    }
   }
   /* OK */
   hts_log_print(opt, LOG_TRACE, "produced temporary name %s", back->tmpfile);
   return 0;
 }
 
-/* Note: utf-8 */
-void back_refetch_backup(httrackp *opt, lien_back *const back) {
-  back->tmpfile = NULL;
-  if (fexist_utf8(back->url_sav)) {
-    hts_boolean saved = HTS_FALSE;
-
-    if (create_back_tmpfile(opt, back, "bak") == 0) {
-      /* clobber a .bak a killed run left behind, or the guard stays off for
-         good (#758) */
-      if (fexist_utf8(back->tmpfile))
-        hts_log_print(opt, LOG_WARNING, "replacing leftover backup %s",
-                      back->tmpfile);
-      saved = hts_rename_over(opt, back->url_sav, back->tmpfile);
-      /* Another slot sharing the directory may have removed it between the
-         structcheck above and the rename: recreate it and try once more. */
-      if (!saved && structcheck(back->tmpfile) == 0)
-        saved = hts_rename_over(opt, back->url_sav, back->tmpfile);
-    }
-    if (!saved) {
-      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
-                    "could not back up %s; an aborted re-fetch will lose it",
-                    back->url_sav);
-      back->tmpfile = NULL;
-    }
-  }
-}
-
-/* Did the fetch fail to produce a response, as opposed to the engine
-   deliberately passing the resource over? Only the latter may be purged. */
-static hts_boolean back_transfer_failed(const int statuscode) {
-  switch (statuscode) {
-  case STATUSCODE_TOO_BIG:
-  case STATUSCODE_EXCLUDED:
-  case STATUSCODE_TEST_OK:
-    return HTS_FALSE;
-  default:
-    return statuscode <= 0 ? HTS_TRUE : HTS_FALSE;
-  }
-}
-
-hts_boolean back_finalize_backup(httrackp *opt, lien_back *const back,
-                                 hts_boolean commit) {
-  const hts_boolean wanted = commit;
-
-  if (back->tmpfile == NULL || back->r.compressed)
+/* Move src onto dst; RENAME does not clobber an existing target on Windows. */
+static hts_boolean replace_file(const char *src, const char *dst) {
+  if (RENAME(src, dst) == 0)
     return HTS_TRUE;
-  /* Nothing to commit to: filecreate() can fail after the backup was taken,
-     and dropping it then loses both copies (#775). */
-  if (commit && !fexist_utf8(back->url_sav)) {
-    hts_log_print(opt, LOG_WARNING, "%s was never created; restoring %s",
-                  back->url_sav, back->tmpfile);
-    commit = HTS_FALSE;
-  }
+  (void) UNLINK(dst);
+  return RENAME(src, dst) == 0 ? HTS_TRUE : HTS_FALSE;
+}
+
+/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
+   existing file moved the good copy to back->tmpfile before truncating url_sav.
+   commit keeps the new file and drops the backup; else restore it so an aborted
+   transfer leaves the previous copy intact. Skips the zlib .z temp. */
+static void back_finalize_backup(httrackp *opt, lien_back *const back,
+                                 const hts_boolean commit) {
+  if (back->tmpfile == NULL || back->r.compressed)
+    return;
   if (commit) {
     (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
-    back_tmpdir_drop(back->tmpfile);
   } else {
     if (back->r.out != NULL) {
       fclose(back->r.out);
@@ -802,21 +615,12 @@ hts_boolean back_finalize_backup(httrackp *opt, lien_back *const back,
     }
     /* On failure keep the backup: an orphaned temp beats losing the good copy.
      */
-    if (!hts_rename_over(opt, back->tmpfile, back->url_sav))
+    if (!replace_file(back->tmpfile, back->url_sav))
       hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
                     "could not restore %s; previous copy kept as %s",
                     back->url_sav, back->tmpfile);
-    else
-      back_tmpdir_drop(back->tmpfile);
   }
   back->tmpfile = NULL;
-  return commit == wanted ? HTS_TRUE : HTS_FALSE;
-}
-
-/* A chunked body is framed by its terminating zero-length chunk (#840);
-   chunk_blocksize is reset per response and reaches -1 only once it is seen. */
-static hts_boolean back_chunked_unterminated(const lien_back *const back) {
-  return back->is_chunk && back->chunk_blocksize != -1 ? HTS_TRUE : HTS_FALSE;
 }
 
 // objet (lien) téléchargé ou transféré depuis le cache
@@ -838,22 +642,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
     /* Don't store broken files. Note: check is done before compression.
        If the file is partial, the next run will attempt to continue it with compression too.
      */
-    const hts_boolean cut_chunked = back_chunked_unterminated(&back[p]);
-    const hts_boolean short_body =
-        back[p].r.totalsize >= 0 && back[p].r.size != back[p].r.totalsize
-            ? HTS_TRUE
-            : HTS_FALSE;
-
-    if ((short_body || cut_chunked) && back[p].r.statuscode > 0 &&
-        !opt->tolerant) {
-      if (cut_chunked) {
-        hts_log_print(
-            opt, LOG_WARNING,
-            "truncated chunked transfer (terminating chunk missing, got " LLintP
-            " bytes): file not cached, will be retried on the next"
-            " update (use -%%B to cache anyway): %s%s",
-            back[p].r.size, back[p].url_adr, back[p].url_fil);
-      } else if (back[p].status == STATUS_READY) {
+    if (back[p].r.totalsize >= 0 && back[p].r.statuscode > 0
+        && back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
+      if (back[p].status == STATUS_READY) {
         hts_log_print(opt, LOG_WARNING,
                       "incomplete transfer (expected " LLintP
                       " bytes, got " LLintP
@@ -931,21 +722,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                     hts_codec_parse(back[p].r.contentencoding);
                 /* Never decode over url_sav: a failed decode would destroy the
                    copy an --update re-fetch is supposed to refresh (#557). */
-                char BIGSTK unpacked[HTS_URLMAXSIZE * 2];
+                char BIGSTK unpacked[HTS_URLMAXSIZE * 2 + 4]; // room for ".u"
                 LLint size;
 
-                /* fits whenever the .z temp it decodes from did */
-                if (!back_tmpname(unpacked, sizeof(unpacked), back[p].url_sav,
-                                  "u")) {
-                  back[p].r.statuscode = STATUSCODE_INVALID;
-                  strcpybuff(back[p].r.msg, "Error when decompressing (the "
-                                            "temporary filename is too long)");
-                  /* as the decode-failure branch below: never let the coded
-                     bytes be committed as the page */
-                  if (!back[p].r.is_write)
-                    deleteaddr(&back[p].r);
-                } else if ((size = hts_codec_unpack(codec, back[p].tmpfile,
-                                                    unpacked)) >= 0) {
+                snprintf(unpacked, sizeof(unpacked), "%s.u", back[p].url_sav);
+                if ((size = hts_codec_unpack(codec, back[p].tmpfile,
+                                             unpacked)) >= 0) {
                   back[p].r.size = back[p].r.totalsize = size;
                   if (back[p].r.is_write) {
                     /* Sample the previous copy now: the rename below replaces
@@ -964,7 +746,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                                  "Read error when decompressing");
                     }
                     UNLINK(unpacked);
-                  } else if (hts_rename_over(opt, unpacked, back[p].url_sav)) {
+                  } else if (replace_file(unpacked, back[p].url_sav)) {
                     /* The temp bypassed filecreate(), which is what chmods. */
 #ifndef _WIN32
                     chmod(back[p].url_sav, HTS_ACCESS_FILE);
@@ -1017,7 +799,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
               /* ensure that no remaining temporary file exists */
               if (back[p].tmpfile != NULL) {
                 unlink(back[p].tmpfile);
-                back_tmpdir_drop(back[p].tmpfile); /* the .u went with it */
                 back[p].tmpfile = NULL;
               }
             }
@@ -1030,14 +811,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
         }
         /* Body fully received: keep the freshly written url_sav, drop the
            backup of the previous copy. */
-        if (!back_finalize_backup(opt, &back[p], HTS_TRUE)) {
-          /* The previous copy is back because the new one was never created;
-             caching this response's validators against it would pin the stale
-             body on every later --update. */
-          if (fexist_utf8(back[p].url_sav))
-            filenote(&opt->state.strc, back[p].url_sav, NULL);
-          return -1;
-        }
+        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -1097,15 +871,16 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           char flags[32];
           char s[256];
           time_t tt;
-          struct tm tmv;
+          struct tm *A;
 
           tt = time(NULL);
-          if (!hts_localtime(tt, &tmv)) {
+          A = localtime(&tt);
+          if (A == NULL) {
             int localtime_returned_null = 0;
 
             assertf(localtime_returned_null);
           }
-          strftime(s, 250, "%H:%M:%S", &tmv);
+          strftime(s, 250, "%H:%M:%S", A);
 
           flags[0] = '\0';
           /* input flags */
@@ -1269,14 +1044,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
   /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
      previous copy from the backup. */
   back_finalize_backup(opt, &back[p], HTS_FALSE);
-  /* Note the surviving copy, or the end-of-update purge drops what this run
-     never managed to replace (#746). */
-  if (!back[p].testmode && back_transfer_failed(back[p].r.statuscode) &&
-      back[p].url_sav[0] != '\0' && fexist_utf8(back[p].url_sav)) {
-    filenote(&opt->state.strc, back[p].url_sav, NULL);
-    file_notify(opt, back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0,
-                back[p].r.notmodified);
-  }
   return -1;
 }
 
@@ -1324,14 +1091,6 @@ void back_connxfr(htsblk * src, htsblk * dst) {
   src->keep_alive_t = 0;
   dst->debugid = src->debugid;
   src->debugid = 0;
-  dst->address = src->address; // peer IP survives the cache-entry swap (#838)
-}
-
-/* Release the buffers a response owns. The connection members are left alone:
-   back_connxfr() moves those, and the file handles are closed elsewhere. */
-static void back_free_response(htsblk *r) {
-  deleteaddr(r);
-  warc_free_request(r);
 }
 
 void back_move(lien_back * src, lien_back * dst) {
@@ -1427,10 +1186,7 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
     (*dst)->r.ssl_con = NULL;
 #endif
     if (back_data_unserialize(fp, (void **) &(*dst)->r.adr, &size) == 0) {
-      /* A bodyless slot already wrote its bytes to url_sav (FTP, direct to
-         disk); zeroing r.size makes the writer blank that file (#797). */
-      if ((*dst)->r.adr != NULL)
-        (*dst)->r.size = size;
+      (*dst)->r.size = size;
       (*dst)->r.headers = NULL;
       if (back_string_unserialize(fp, &(*dst)->r.headers) == 0)
         return 0;               /* ok */
@@ -1869,7 +1625,10 @@ int back_clear_entry(lien_back * back) {
       back->r.soc = INVALID_SOCKET;
     }
 
-    back_free_response(&back->r);
+    if (back->r.adr != NULL) {  // reste un bloc à désallouer
+      freet(back->r.adr);
+      back->r.adr = NULL;
+    }
     if (back->chunk_adr != NULL) {      // reste un bloc à désallouer
       freet(back->chunk_adr);
       back->chunk_adr = NULL;
@@ -1880,9 +1639,14 @@ int back_clear_entry(lien_back * back) {
     // only for security
     if (back->tmpfile && back->tmpfile[0] != '\0') {
       (void) unlink(back->tmpfile);
-      back_tmpdir_drop(back->tmpfile);
       back->tmpfile = NULL;
     }
+    // headers
+    if (back->r.headers != NULL) {
+      freet(back->r.headers);
+      back->r.headers = NULL;
+    }
+    warc_free_request(&back->r);
     // Tout nettoyer
     memset(back, 0, sizeof(lien_back));
     back->r.soc = INVALID_SOCKET;
@@ -2180,8 +1944,6 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
                             back[p].url_adr, back[p].url_fil);
             }
             back[p].r.notmodified = 1;  // fichier non modifié
-            // no request was sent at all, so this is never a server 304 (#839)
-            back[p].r.warc_forced_notmodified = HTS_TRUE;
             back[p].status = STATUS_READY; // OK prêt
             back_set_finished(sback, p);
 
@@ -3394,7 +3156,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
-                              back_refetch_backup(opt, &back[i]);
+                              /* Re-fetch over an existing file (#77 follow-up):
+                                 move the good copy aside before truncating it
+                                 so an aborted transfer can restore it. url_sav
+                                 is still written normally (file list intact).
+                               */
+                              back[i].tmpfile = NULL;
+                              if (fexist_utf8(back[i].url_sav)) {
+                                if (create_back_tmpfile(opt, &back[i], "bak") !=
+                                        0 ||
+                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
+                                        0) {
+                                  back[i].tmpfile = NULL;
+                                }
+                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3577,26 +3352,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back_finalize(opt, cache, sback, i);
             }
 
-            /* Same treatment for an unterminated chunked stream: the byte count
-               agrees with the chunks that arrived, the framing does not. */
-            if (back[i].r.statuscode > 0 &&
-                back_chunked_unterminated(&back[i])) {
-              if (!opt->tolerant) {
-                deleteaddr(&back[i].r);
-                back[i].r.statuscode = STATUSCODE_CONNERROR; // recatch
-                htsblk_failf(&back[i].r,
-                             "Truncated chunked transfer (" LLintP
-                             " Bytes, terminating chunk missing)",
-                             (LLint) back[i].r.size);
-              } else {
-                hts_log_print(opt, LOG_WARNING,
-                              "Truncated chunked transfer (" LLintP
-                              " Bytes, terminating chunk missing) for %s%s",
-                              (LLint) back[i].r.size, back[i].url_adr,
-                              back[i].url_fil);
-              }
-            }
-
             if (back[i].r.totalsize >= 0) { // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
@@ -3656,22 +3411,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.totalsize < 0)
                     back[i].r.totalsize = 0;    // initialiser à 0 (-1 == unknown)
                   if (back[i].status == STATUS_CHUNK_WAIT) {    // "real" chunk
-                    /* The chunk-size line is hostile input, so parse it wide
-                       and unsigned and drop anything an int cannot hold: sscanf
-                       "%x" lands 80000000 on INT_MIN, which sign-extends into a
-                       16EB realloc below and drives totalsize negative. */
-                    char *chunk_end = NULL;
-                    const unsigned long long chunk_value =
-                        strtoull(chunk_data, &chunk_end, 16);
-
-                    if (chunk_end != chunk_data && *chunk_end == '\0' &&
-                        chunk_value <= (unsigned long long) INT32_MAX) {
-                      chunk_size = (int) chunk_value;
+                    if (sscanf(chunk_data, "%x", &chunk_size) == 1) {
                       if (chunk_size > 0)
                         back[i].chunk_blocksize = chunk_size;   /* the data block chunk size */
-                      /* only a real 0 ends the stream; the bound above keeps a
-                         negative from ever claiming the sentinel (#840) */
-                      else if (chunk_size == 0)
+                      else
                         back[i].chunk_blocksize = -1;   /* ending */
                       back[i].r.totalsize += chunk_size;        // noter taille
                       if (back[i].r.adr != NULL || !back[i].r.is_write) {       // Not to disk
@@ -4221,20 +3964,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
                       memset(&tmp, 0, sizeof(tmp));
                       back_connxfr(&back[i].r, &tmp);
-                      /* a real 304's headers belong to the revisit record, so
-                         they must survive the swap (#826); a forced one has
-                         none */
-                      if (server_sent_304)
-                        warc_move_request(&back[i].r, &tmp);
-                      /* the cache entry overwrites the whole struct, so drop
-                         what the 304 response still owns first (#782) */
-                      back_free_response(&back[i].r);
                       back[i].r =
                         cache_read(opt, cache, back[i].url_adr, back[i].url_fil,
                                    back[i].url_sav, back[i].location_buffer);
                       back[i].r.location = back[i].location_buffer;
                       back_connxfr(&tmp, &back[i].r);
-                      warc_move_request(&tmp, &back[i].r);
                     }
 
                     // hack:
@@ -4260,9 +3994,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       back[i].status = STATUS_READY;    // OK prêt
                       back_set_finished(sback, i);
                       back[i].r.notmodified = 1;        // NON modifié!
-                      // WARC must not claim a 304 the server never sent (#839)
-                      back[i].r.warc_forced_notmodified =
-                          server_sent_304 ? HTS_FALSE : HTS_TRUE;
                       hts_log_print(opt, LOG_DEBUG,
                                     "File loaded after test from cache: %s%s",
                                     back[i].url_adr, back[i].url_fil);

src/htsback.h

@@ -139,24 +139,6 @@ int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback,
                  const int p);
 int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                   const int p);
-/* Move the previous copy of back->url_sav to back->tmpfile so back_finalize()
-   can put it back when the re-fetch fails (#77 follow-up). Call right before
-   truncating url_sav; tmpfile stays NULL when there is nothing to save. */
-void back_refetch_backup(httrackp *opt, lien_back *const back);
-/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
-   existing file moved the good copy to back->tmpfile before truncating url_sav.
-   commit keeps the new file and drops the backup, unless url_sav was never
-   created; else restore it so an aborted transfer leaves the previous copy
-   intact. Skips the zlib .z temp. HTS_FALSE when a requested commit had to
-   restore instead: the caller then holds the OLD body and must not cache this
-   response's validators against it. */
-hts_boolean back_finalize_backup(httrackp *opt, lien_back *const back,
-                                 hts_boolean commit);
-/* Remove the reserved directory a temporary sat in, once the last slot sharing
-   it is done; a non-empty one just refuses. No-op outside that directory. */
-void back_tmpdir_drop(const char *tmp);
-/* -#test=backswap: slots eligible for the on-disk ready table. */
-int back_selftest_slot_swap(void);
 void back_info(struct_back * sback, int i, int j, FILE * fp);
 void back_infostr(struct_back *sback, int i, int j, char *s, size_t size);
 LLint back_transferred(LLint add, struct_back * sback);

src/htsbauth.c

@@ -491,7 +491,7 @@ char *bauth_prefix(char *prefix, const char *adr, const char *fil) {
   if (a)
     *a = '\0';
   if (strchr(prefix, '/')) {
-    a = hts_lastcharptr(prefix);
+    a = prefix + strlen(prefix) - 1;
     while(*a != '/')
       a--;
     *(a + 1) = '\0';

src/htscache.c

@@ -424,11 +424,6 @@ htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
 
     if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
       r = itemback->r;
-      /* header fields only, like cache_readex(): the entry torn down below
-         owns these (#826) */
-      r.adr = NULL;
-      r.headers = NULL;
-      r.location = NULL;
       if (return_save != NULL)
         strlcpybuff(return_save, itemback->url_sav, HTS_URLMAXSIZE * 2);
       /* cleanup */
@@ -860,6 +855,13 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
   return r;
 }
 
+// lecture d'un fichier dans le cache
+// si save==null alors test unqiquement
+static int hts_rename(httrackp * opt, const char *a, const char *b) {
+  hts_log_print(opt, LOG_DEBUG, "Cache: rename %s -> %s (%p %p)", a, b, a, b);
+  return RENAME(a, b);
+}
+
 /* Open the cache ZIP via hts_fopen_utf8 so a non-ASCII path_log isn't mangled
    to ANSI (#630); 64-bit funcs keep multi-GB caches whole on Windows LLP64. */
 static voidpf ZCALLBACK hts_zip_fopen_utf8(voidpf opaque, const void *filename,
@@ -962,35 +964,6 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
     return NULL;
 }
 
-const char *cache_repair(httrackp *opt, const char *name,
-                         unsigned long *entries, unsigned long *bytes) {
-  char BIGSTK repairname[HTS_URLMAXSIZE * 2];
-  unzFile zip;
-
-  *entries = 0;
-  *bytes = 0;
-  if (!slprintfbuff(repairname, sizeof(repairname), "%s%s",
-                    StringBuff(opt->path_log), "hts-cache/repair.zip"))
-    return "the repair path is too long";
-  if (unzRepair(name, repairname,
-                fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                        StringBuff(opt->path_log), "hts-cache/repair.tmp"),
-                entries, bytes) != Z_OK)
-    return "could not repair the cache";
-  /* unzRepair writes an end-of-central-directory record whatever it found, so
-     an input holding no local file header at all yields a valid empty archive
-     and a short write yields a truncated one. Only an archive that holds
-     something and opens may replace the cache (#824). */
-  if (*entries == 0)
-    return "the repaired cache holds no entry, keeping the damaged one";
-  if ((zip = hts_unzOpen_utf8(repairname)) == NULL)
-    return "the repaired cache does not open, keeping the damaged one";
-  unzClose(zip);
-  if (!hts_rename_over(opt, repairname, name))
-    return "could not put the repaired cache in place";
-  return NULL;
-}
-
 // Initialisation du cache: créer nouveau, renomer ancien, charger..
 void cache_init(cache_back * cache, httrackp * opt) {
   // ---
@@ -1018,16 +991,29 @@ void cache_init(cache_back * cache, httrackp * opt) {
               OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
               StringBuff(opt->path_log),
               "hts-cache/new.zip")))) { // a previous cache exists.. rename it
-        if (!hts_rename_over(
-                opt,
-                fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                        StringBuff(opt->path_log), "hts-cache/new.zip"),
-                fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                        StringBuff(opt->path_log), "hts-cache/old.zip"))) {
+        /* Remove OLD cache */
+        if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                StringBuff(opt->path_log),
+                                "hts-cache/old.zip"))) {
+          if (UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                             StringBuff(opt->path_log), "hts-cache/old.zip")) !=
+              0) {
+            hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                          "Cache: error while moving previous cache");
+          }
+        }
+
+        /* Rename */
+        if (hts_rename
+            (opt,
+             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                     "hts-cache/new.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                   StringBuff(opt->path_log),
+                                                   "hts-cache/old.zip")) != 0) {
           hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
                         "Cache: error while moving previous cache");
         } else {
-          hts_log_print(opt, LOG_DEBUG, "Cache: rotated new.zip to old.zip");
+          hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
         }
       }
     } else {
@@ -1060,9 +1046,8 @@ void cache_init(cache_back * cache, httrackp * opt) {
       // Corrupted ZIP file ? Try to repair!
       if (cache->zipInput == NULL && !cache->ro) {
         char *name;
-        const char *why;
-        unsigned long repaired = 0;
-        unsigned long repairedBytes = 0;
+        uLong repaired = 0;
+        uLong repairedBytes = 0;
 
         if (!cache->ro) {
           name =
@@ -1075,16 +1060,25 @@ void cache_init(cache_back * cache, httrackp * opt) {
         }
         hts_log_print(opt, LOG_WARNING,
                       "Cache: damaged cache, trying to repair");
-        why = cache_repair(opt, name, &repaired, &repairedBytes);
-        if (why != NULL) {
-          hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "Cache: %s", why);
-        } else if ((cache->zipInput = hts_unzOpen_utf8(name)) != NULL) {
+        /* mztools has no UTF-8 hook, so repairing a corrupt cache under a
+           non-ASCII path_log fails cleanly (re-crawl), never forks a twin. */
+        if (unzRepair
+            (name,
+             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                     "hts-cache/repair.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                      StringBuff(opt->path_log),
+                                                      "hts-cache/repair.tmp"),
+             &repaired, &repairedBytes) == Z_OK) {
+          UNLINK(name);
+          RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                         StringBuff(opt->path_log), "hts-cache/repair.zip"),
+                 name);
+          cache->zipInput = hts_unzOpen_utf8(name);
           hts_log_print(opt, LOG_WARNING,
                         "Cache: %d bytes successfully recovered in %d entries",
                         (int) repairedBytes, (int) repaired);
         } else {
-          hts_log_print(opt, LOG_WARNING,
-                        "Cache: the repaired cache could not be reopened");
+          hts_log_print(opt, LOG_WARNING, "Cache: could not repair the cache");
         }
       }
       // Opened ?

src/htscache.h

@@ -67,9 +67,7 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
 htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, const char *save, char *location);
 /* Like cache_read, but also yields entries whose transfer broke; return_save
-   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name.
-   Header fields only: adr, headers and location come back NULL, so the caller
-   owns and frees nothing. */
+   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
 htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
                                    const char *adr, const char *fil,
                                    char *return_save);
@@ -80,14 +78,6 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
-/* Recover the damaged cache at name into hts-cache/repair.zip and move it over
-   name, storing what was recovered in *entries and *bytes. Returns NULL on
-   success, else a reason the caller reports: a recovery that is empty or does
-   not open never replaces the cache, and neither does one that cannot be moved
-   into place (#786, #824). Note: utf-8. */
-const char *cache_repair(httrackp *opt, const char *name,
-                         unsigned long *entries, unsigned long *bytes);
-
 /* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
 typedef enum {
   CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */

src/htscache_selftest.c

@@ -309,72 +309,6 @@ static int disk_fallback_selftest(httrackp *opt) {
   return fail;
 }
 
-/* A cache miss sends cache_read_including_broken() to the serialized reference
-   of an interrupted transfer, whose entry it then frees: what it returns must
-   not point into that entry (#826). */
-static int broken_ref_selftest(httrackp *opt) {
-  int fail = 0;
-  cache_back cache;
-  lien_back *entry;
-  htsblk r;
-  char save[HTS_URLMAXSIZE * 2];
-  const char *const adr = "example.com";
-  const char *const fil = "/interrupted.html";
-  static const char body[] = "<html><body>half a p";
-  static const char headers[] =
-      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n";
-
-  entry = calloct(1, sizeof(lien_back));
-  strcpybuff(entry->url_adr, adr);
-  strcpybuff(entry->url_fil, fil);
-  concat(entry->url_sav, sizeof(entry->url_sav),
-         StringBuff(opt->path_html_utf8), "example.com/interrupted.html");
-  hts_init_htsblk(&entry->r);
-  entry->r.statuscode = 200;
-  entry->r.size = (LLint) (sizeof(body) - 1);
-  strcpybuff(entry->r.msg, "OK");
-  strcpybuff(entry->r.contenttype, "text/html");
-  entry->r.location = entry->location_buffer;
-  entry->r.adr = strdupt(body);
-  entry->r.headers = strdupt(headers);
-  if (back_serialize_ref(opt, entry) != 0) {
-    fprintf(stderr, "%s: broken-ref: cannot write the reference\n",
-            selftest_tag);
-    fail++;
-  }
-  freet(entry->r.adr);
-  freet(entry->r.headers);
-  freet(entry);
-
-  selftest_open_for_read(&cache, opt);
-  save[0] = '\0';
-  r = cache_read_including_broken(opt, &cache, adr, fil, save);
-  selftest_close(&cache);
-
-  if (r.statuscode != 200 || strcmp(r.contenttype, "text/html") != 0) {
-    fprintf(stderr,
-            "%s: broken-ref: statuscode %d type '%s', want 200/text/html"
-            " (reference not read back)\n",
-            selftest_tag, r.statuscode, r.contenttype);
-    fail++;
-  }
-  if (strstr(save, "interrupted.html") == NULL) {
-    fprintf(stderr, "%s: broken-ref: save name '%s' lost\n", selftest_tag,
-            save);
-    fail++;
-  }
-  /* the entry owned all three and is gone, so a non-NULL one is dangling */
-  if (r.adr != NULL || r.headers != NULL || r.location != NULL) {
-    fprintf(stderr,
-            "%s: broken-ref: returned freed adr/headers/location"
-            " (%p/%p/%p)\n",
-            selftest_tag, (void *) r.adr, (void *) r.headers,
-            (void *) r.location);
-    fail++;
-  }
-  return fail;
-}
-
 typedef struct {
   size_t budget;  /**< bytes allowed through before writes start failing */
   int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
@@ -742,7 +676,7 @@ int cache_selftests(httrackp *opt, const char *dir) {
     char base[HTS_URLMAXSIZE];
 
     strcpybuff(base, dir);
-    if (base[0] != '\0' && hts_lastchar(base) != '/') {
+    if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
       strcatbuff(base, "/");
     }
     StringCopy(opt->path_log, base);
@@ -866,9 +800,6 @@ int cache_selftests(httrackp *opt, const char *dir) {
   /* pass 5: the disk-fallback read path (X-In-Cache: 0, body on disk) */
   failures += disk_fallback_selftest(opt);
 
-  /* pass 6: the broken-transfer reference fallback */
-  failures += broken_ref_selftest(opt);
-
   for (i = 0; i < large_count; i++) {
     freet(large_body[i]);
   }
@@ -925,7 +856,7 @@ static void golden_setup(httrackp *opt, const char *dir) {
   char base[HTS_URLMAXSIZE];
 
   strcpybuff(base, dir);
-  if (base[0] != '\0' && hts_lastchar(base) != '/') {
+  if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
     strcatbuff(base, "/");
   }
   StringCopy(opt->path_log, base);

src/htscharset.c

@@ -296,41 +296,16 @@ LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int *pwsize) {
   return hts_convertStringToUCS2(s, size, CP_UTF8, pwsize);
 }
 
-/* WideCharToMultiByte rejects lpUsedDefaultChar on the Unicode, ISO-2022, HZ,
-   GB18030 and ISCII codepages, where a substitution stays invisible. */
-static hts_boolean cp_reports_default_char(UINT cp) {
-  if (cp == 42 /* CP_SYMBOL */ || cp == CP_UTF7 || cp == CP_UTF8 ||
-      cp == 52936 || cp == 54936 || (cp >= 50220 && cp <= 50229) ||
-      (cp >= 57002 && cp <= 57011)) {
-    return HTS_FALSE;
-  }
-  return HTS_TRUE;
-}
-
-/* When plossy is non-NULL, *plossy reports that the codepage lacked a character
-   and a substitute was emitted for it. */
-static char *hts_convertUCS2StringToCPEx(LPWSTR woutput, int wsize, UINT cp,
-                                         hts_boolean *plossy) {
+char *hts_convertUCS2StringToCP(LPWSTR woutput, int wsize, UINT cp) {
   const int usize =
-      WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, NULL);
-
-  if (plossy != NULL) {
-    *plossy = HTS_FALSE;
-  }
+    WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, FALSE);
   if (usize > 0) {
     char *const uoutput = malloc((usize + 1) * sizeof(char));
 
     if (uoutput != NULL) {
-      BOOL usedDefault = FALSE;
-      LPBOOL const pUsedDefault =
-          plossy != NULL && cp_reports_default_char(cp) ? &usedDefault : NULL;
-
-      if (WideCharToMultiByte(cp, 0, woutput, wsize, uoutput, usize, NULL,
-                              pUsedDefault) == usize) {
+      if (WideCharToMultiByte
+          (cp, 0, woutput, wsize, uoutput, usize, NULL, FALSE) == usize) {
         uoutput[usize] = '\0';
-        if (plossy != NULL && usedDefault) {
-          *plossy = HTS_TRUE;
-        }
         return uoutput;
       } else {
         free(uoutput);
@@ -340,10 +315,6 @@ static char *hts_convertUCS2StringToCPEx(LPWSTR woutput, int wsize, UINT cp,
   return NULL;
 }
 
-char *hts_convertUCS2StringToCP(LPWSTR woutput, int wsize, UINT cp) {
-  return hts_convertUCS2StringToCPEx(woutput, wsize, cp, NULL);
-}
-
 char *hts_convertUCS2StringToUTF8(LPWSTR woutput, int wsize) {
   return hts_convertUCS2StringToCP(woutput, wsize, CP_UTF8);
 }
@@ -375,11 +346,7 @@ char *hts_convertStringCPToUTF8(const char *s, size_t size, UINT cp) {
   return NULL;
 }
 
-static char *hts_convertStringCPFromUTF8Ex(const char *s, size_t size, UINT cp,
-                                           hts_boolean *plossy) {
-  if (plossy != NULL) {
-    *plossy = HTS_FALSE;
-  }
+char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
   /* Empty string ? */
   if (size == 0) {
     return hts_stringMemCopy(s, size);
@@ -395,8 +362,7 @@ static char *hts_convertStringCPFromUTF8Ex(const char *s, size_t size, UINT cp,
     LPWSTR woutput = hts_convertStringToUCS2(s, (int) size, CP_UTF8, &wsize);
 
     if (woutput != NULL) {
-      char *const uoutput =
-          hts_convertUCS2StringToCPEx(woutput, wsize, cp, plossy);
+      char *const uoutput = hts_convertUCS2StringToCP(woutput, wsize, cp);
 
       free(woutput);
       return uoutput;
@@ -407,10 +373,6 @@ static char *hts_convertStringCPFromUTF8Ex(const char *s, size_t size, UINT cp,
   return NULL;
 }
 
-char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
-  return hts_convertStringCPFromUTF8Ex(s, size, cp, NULL);
-}
-
 HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
                                          const char *charset) {
   const UINT cp = hts_getCodepage(charset);
@@ -424,19 +386,6 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
   return hts_convertStringCPFromUTF8(s, size, cp);
 }
 
-char *hts_convertStringFromUTF8Strict(const char *s, size_t size,
-                                      const char *charset) {
-  hts_boolean lossy = HTS_FALSE;
-  char *const out =
-      hts_convertStringCPFromUTF8Ex(s, size, hts_getCodepage(charset), &lossy);
-
-  if (lossy) {
-    free(out);
-    return NULL;
-  }
-  return out;
-}
-
 HTSEXT_API char *hts_convertStringSystemToUTF8(const char *s, size_t size) {
   return hts_convertStringCPToUTF8(s, size, GetACP());
 }
@@ -641,13 +590,6 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
   }
 }
 
-char *hts_convertStringFromUTF8Strict(const char *s, size_t size,
-                                      const char *charset) {
-  /* No transliteration is requested of iconv, so an unrepresentable code point
-     already fails the conversion outright. */
-  return hts_convertStringFromUTF8(s, size, charset);
-}
-
 #endif
 
 #ifdef _WIN32

src/htscharset.h

@@ -61,15 +61,6 @@ HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
 extern char *hts_convertStringFromUTF8(const char *s, size_t size,
                                        const char *charset);
 
-/**
- * Same, but refusing to lose a code point the charset can not represent:
- * NULL is returned then, where hts_convertStringFromUTF8() may hand back a
- * substituted string (Windows substitutes, iconv fails).
- * Return NULL upon error.
- **/
-extern char *hts_convertStringFromUTF8Strict(const char *s, size_t size,
-                                             const char *charset);
-
 /**
  * Convert an UTF-8 string to an IDNA (RFC 3492) string.
  **/

src/htscodec.c

@@ -72,7 +72,7 @@ hts_codec hts_codec_parse(const char *encoding) {
     return HTS_CODEC_IDENTITY;
   if (strfield2(encoding, "gzip") || strfield2(encoding, "x-gzip") ||
       strfield2(encoding, "deflate") || strfield2(encoding, "x-deflate"))
-    return HTS_CODEC_DEFLATE;
+    return HTS_USEZLIB ? HTS_CODEC_DEFLATE : HTS_CODEC_UNSUPPORTED;
   if (strfield2(encoding, "br"))
     return HTS_USEBROTLI ? HTS_CODEC_BROTLI : HTS_CODEC_UNSUPPORTED;
   if (strfield2(encoding, "zstd"))
@@ -98,11 +98,16 @@ hts_codec hts_codec_parse(const char *encoding) {
 const char *hts_acceptencoding(hts_boolean compressible, hts_boolean secure) {
   if (!compressible)
     return "identity";
+#if HTS_USEZLIB
   /* br and zstd over TLS only, as browsers do: a cleartext intermediary that
      rewrites a coding it can not read would corrupt the mirror. */
   if (secure)
     return "gzip, deflate" HTS_AE_BROTLI HTS_AE_ZSTD ", identity;q=0.9";
   return "gzip, deflate, identity;q=0.9";
+#else
+  (void) secure;
+  return "identity";
+#endif
 }
 
 hts_boolean hts_codec_is_archive_ext(hts_codec codec, const char *ext) {
@@ -295,7 +300,11 @@ int hts_codec_unpack(hts_codec codec, const char *filename,
     return -1;
   switch (codec) {
   case HTS_CODEC_DEFLATE:
+#if HTS_USEZLIB
     return hts_zunpack(filename, newfile);
+#else
+    return -1;
+#endif
   case HTS_CODEC_BROTLI:
   case HTS_CODEC_ZSTD:
     break;
@@ -339,8 +348,10 @@ size_t hts_codec_head(hts_codec codec, const void *in, size_t in_len, void *out,
   if (in == NULL || in_len == 0 || out == NULL || out_len == 0)
     return 0;
   switch (codec) {
+#if HTS_USEZLIB
   case HTS_CODEC_DEFLATE:
     return hts_zhead(in, in_len, out, out_len);
+#endif
 #if HTS_USEBROTLI
   case HTS_CODEC_BROTLI:
     return codec_head_brotli(in, in_len, out, out_len);

src/htscore.c

@@ -39,7 +39,6 @@ Please visit our Website: http://www.httrack.com
 
 /* File defs */
 #include "htscore.h"
-#include "htssitemap.h"
 #include "htswarc.h"
 #include "htschanges.h"
 #include "htssinglefile.h"
@@ -773,7 +772,7 @@ int httpmirror(char *url1, httrackp * opt) {
         // sauter les + sans rien après..
         if (strnotempty(tempo)) {
           if ((plus == 0) && (type == 1)) {     // implicite: *www.edf.fr par exemple
-            if (hts_lastchar(tempo) != '*') {
+            if (tempo[strlen(tempo) - 1] != '*') {
               strcatbuff(tempo, "*");   // ajouter un *
             }
           }
@@ -950,22 +949,6 @@ int httpmirror(char *url1, httrackp * opt) {
     heap_top()->premier = heap_top_index();        // premier lien, objet-père=objet              
     heap_top()->precedent = heap_top_index();      // lien précédent
 
-    /* --sitemap: queue the sitemap probe just after the seeds, so its URLs are
-       injected before the crawl gets far. */
-    hts_sitemap_free(opt); /* an earlier mirror may have left a doc list */
-    if (opt->sitemap || StringNotEmpty(opt->sitemap_url)) {
-      char BIGSTK first[HTS_URLMAXSIZE * 2];
-      const char *const eol = strchr(primary, '\n');
-      const size_t len = eol != NULL ? (size_t) (eol - primary) : 0;
-
-      first[0] = '\0';
-      if (len > 0 && len < sizeof(first)) {
-        memcpy(first, primary, len);
-        first[len] = '\0';
-      }
-      hts_sitemap_seed(opt, first);
-    }
-
     // Initialiser cache
     {
       opt->state._hts_in_html_parsing = 4;
@@ -1045,14 +1028,13 @@ int httpmirror(char *url1, httrackp * opt) {
     {
       TStamp tl = 0;
       time_t tt;
-      struct tm tmv;
+      struct tm *A;
 
       tt = time(NULL);
-      if (hts_localtime(tt, &tmv)) {
-        tl += tmv.tm_sec;
-        tl += tmv.tm_min * 60;
-        tl += tmv.tm_hour * 60 * 60;
-      }
+      A = localtime(&tt);
+      tl += A->tm_sec;
+      tl += A->tm_min * 60;
+      tl += A->tm_hour * 60 * 60;
       if (tl > opt->waittime)   // attendre minuit
         rollover = 1;
     }
@@ -1062,14 +1044,13 @@ int httpmirror(char *url1, httrackp * opt) {
     do {
       TStamp tl = 0;
       time_t tt;
-      struct tm tmv;
+      struct tm *A;
 
       tt = time(NULL);
-      if (hts_localtime(tt, &tmv)) {
-        tl += tmv.tm_sec;
-        tl += tmv.tm_min * 60;
-        tl += tmv.tm_hour * 60 * 60;
-      }
+      A = localtime(&tt);
+      tl += A->tm_sec;
+      tl += A->tm_min * 60;
+      tl += A->tm_hour * 60 * 60;
 
       if (rollover) {
         if (tl <= opt->waittime)
@@ -1612,21 +1593,11 @@ int httpmirror(char *url1, httrackp * opt) {
         stre.maketrack_fp = maketrack_fp;
 
         /* Parse */
-        {
-          const int nlinks = opt->lien_tot;
-
-          if (hts_mirror_check_moved(&str, &stre) != 0) {
-            XH_uninit;
-            return -1;
-          }
-          /* A redirect re-queues the target as a fresh link; without carrying
-             the marking over, a moved sitemap is fetched and then ignored. */
-          if (opt->sitemap_state != NULL && opt->lien_tot > nlinks &&
-              hts_sitemap_pending(opt, urladr(), urlfil())) {
-            hts_sitemap_redirect(opt, urladr(), urlfil(), heap_top()->adr,
-                                 heap_top()->fil);
-          }
+        if (hts_mirror_check_moved(&str, &stre) != 0) {
+          XH_uninit;
+          return -1;
         }
+
       }
 
     }                           // if !error
@@ -1644,29 +1615,6 @@ int httpmirror(char *url1, httrackp * opt) {
       /* Load file and decode if necessary, after redirect check. */
       LOAD_IN_MEMORY_IF_NECESSARY();
 
-      /* Sitemap document: turn its <loc> URLs into top-level seeds. They go
-         through htsAddLink, so the wizard's filters and scope rules decide, and
-         this link's max depth leaves them the full budget. */
-      if (opt->sitemap_state != NULL &&
-          hts_sitemap_pending(opt, urladr(), urlfil())) {
-        htsmoduleStruct BIGSTK smstr;
-        int smptr = ptr;
-
-        memset(&smstr, 0, sizeof(smstr));
-        smstr.opt = opt;
-        smstr.sback = sback;
-        smstr.cache = &cache;
-        smstr.hashptr = hashptr;
-        smstr.numero_passe = numero_passe;
-        smstr.ptr_ = &smptr; /* scratch: the ingester retargets the wizard */
-        smstr.addLink = htsAddLink;
-        smstr.url_host = urladr();
-        smstr.url_file = urlfil();
-        smstr.mime = r.contenttype;
-        hts_sitemap_ingest(opt, &smstr, urladr(), urlfil(), r.adr,
-                           r.adr != NULL && r.size > 0 ? (size_t) r.size : 0);
-      }
-
       // ------------------------------------------------------
       // ok, fichier chargé localement
       // ------------------------------------------------------
@@ -1730,12 +1678,16 @@ int httpmirror(char *url1, httrackp * opt) {
               if (charset != NULL)
                 free(charset);
             }
-            /* Left empty when the document declared none: guessing UTF-8 here
-               would break links, and the parser picks its own fallback */
+            /* Could not detect charset: could it be UTF-8 ? */
+            /* No, we can not do that: browsers do not do it
+               (and it would break links). */
+            /* Could not detect charset */
             if (page_charset[0] == '\0') {
               hts_log_print(opt, LOG_INFO,
                             "Warning: could not detect encoding for: %s%s",
                             urladr(), urlfil());
+              /* Fallback to ISO-8859-1 (~== identity) ; accents will look weird */
+              strcpy(page_charset, "iso-8859-1");
             }
           }
 
@@ -1768,9 +1720,7 @@ int httpmirror(char *url1, httrackp * opt) {
             /* */
             str.ptr_ = &ptr;
             /* */
-            /* NULL when conversion is off, empty when the document declared no
-               charset; the parser tells the two apart */
-            str.page_charset_ = opt->convert_utf8 ? page_charset : NULL;
+            str.page_charset_ = page_charset[0] != '\0' ? page_charset : NULL;
             /* */
             /* */
             stre.r_ = &r;
@@ -1819,11 +1769,62 @@ int httpmirror(char *url1, httrackp * opt) {
         // -- -- --
         // sauver fichier
 
+        /* En cas d'erreur, vérifier que fichier d'erreur existe */
+        if (strnotempty(savename()) == 0) {       // chemin de sauvegarde existant
+          if (strcmp(urlfil(), "/robots.txt") == 0) {     // pas robots.txt
+            if (store_errpage) {        // c'est une page d'erreur
+              int create_html_warning = 0;
+              int create_gif_warning = 0;
+
+              switch (ishtml(opt, urlfil())) {    /* pas fichier html */
+              case 0:          /* non html */
+                {
+                  char buff[256];
+
+                  guess_httptype_sized(opt, buff, sizeof(buff), urlfil());
+                  if (strcmp(buff, "image/gif") == 0)
+                    create_gif_warning = 1;
+                }
+                break;
+              case 1:          /* html */
+                if (!r.adr) {
+                }
+                break;
+              default:         /* don't know.. */
+                break;
+              }
+              /* Créer message d'erreur ? */
+              if (create_html_warning) {
+                char *adr =
+                  (char *) malloct(strlen(HTS_DATA_ERROR_HTML) + 1100);
+                hts_log_print(opt, LOG_INFO, "Creating HTML warning file (%s)",
+                              r.msg);
+                if (adr) {
+                  if (r.adr) {
+                    freet(r.adr);
+                    r.adr = NULL;
+                  }
+                  sprintf(adr, HTS_DATA_ERROR_HTML, r.msg);
+                  r.adr = adr;
+                }
+              } else if (create_gif_warning) {
+                char *adr = (char *) malloct(HTS_DATA_UNKNOWN_GIF_LEN);
+
+                hts_log_print(opt, LOG_INFO, "Creating GIF dummy file (%s)",
+                              r.msg);
+                if (r.adr) {
+                  freet(r.adr);
+                  r.adr = NULL;
+                }
+                memcpy(adr, HTS_DATA_UNKNOWN_GIF, HTS_DATA_UNKNOWN_GIF_LEN);
+                r.adr = adr;
+              }
+            }
+          }
+        }
+
         if (strnotempty(savename()) == 0) {       // pas de chemin de sauvegarde
           if (strcmp(urlfil(), "/robots.txt") == 0) {     // robots.txt
-            char BIGSTK sitemaps[8192];
-
-            sitemaps[0] = '\0';
             if (r.adr) {
               char BIGSTK infobuff[8192];
 #ifdef IGNORE_RESTRICTIVE_ROBOTS
@@ -1835,8 +1836,7 @@ int httpmirror(char *url1, httrackp * opt) {
 #endif
 
               robots_parse(&robots, urladr(), r.adr, r.size, infobuff,
-                           sizeof(infobuff), keep_root, sitemaps,
-                           sizeof(sitemaps));
+                           sizeof(infobuff), keep_root);
               if (strnotempty(infobuff)) {
                 hts_log_print(opt, LOG_INFO,
                               "Note: robots.txt forbidden links for %s are: %s",
@@ -1846,10 +1846,6 @@ int httpmirror(char *url1, httrackp * opt) {
                               urladr(), infobuff);
               }
             }
-            /* After robots_parse, so the rules this very body carries already
-               gate the sitemap fetch. Runs even on a failed probe, which is
-               what falls back to the well-known location. */
-            hts_sitemap_robots(opt, urladr(), sitemaps);
           }
         } else if (r.is_write) {        // déja sauvé sur disque
           /*
@@ -1926,9 +1922,10 @@ int httpmirror(char *url1, httrackp * opt) {
           }
 
           // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-          // A failed transfer has no body: r.adr holds debris from the aborted
-          // read, which would destroy the copy being re-fetched (#748).
-          if (r.statuscode > 0 && (r.adr != NULL || r.size == 0)) {
+          // An empty body must not overwrite the file when the transfer failed
+          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
+          // copy to 0 (#77 follow-up).
+          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -2136,9 +2133,7 @@ int httpmirror(char *url1, httrackp * opt) {
                   continue;
                 strcpybuff(file, StringBuff(opt->path_html));
                 strcatbuff(file, line + 1);
-                /* strip filenote()'s ']', absent when linput() truncated the
-                   line */
-                hts_striplastchar(file, ']');
+                file[strlen(file) - 1] = '\0';
                 hts_changes_previous(opt, file + StringLength(opt->path_html));
                 if (!strstr(adr, line)) { // not found in the new list?
                   if (fexist_utf8(file)) { // still on disk
@@ -2165,11 +2160,12 @@ int httpmirror(char *url1, httrackp * opt) {
                 fseek(old_lst, 0, SEEK_SET);
                 while(!feof(old_lst)) {
                   linput(old_lst, line, 1000);
-                  while (strnotempty(line) && (hts_lastchar(line) != '/') &&
-                         (hts_lastchar(line) != '\\')) {
-                    hts_choplastchar(line);
+                  while(strnotempty(line) && (line[strlen(line) - 1] != '/')
+                        && (line[strlen(line) - 1] != '\\')) {
+                    line[strlen(line) - 1] = '\0';
                   }
-                  hts_choplastchar(line);
+                  if (strnotempty(line))
+                    line[strlen(line) - 1] = '\0';
                   if (strnotempty(line))
                     if (!strstr(adr, line)) {   // non trouvé?
                       char BIGSTK file[HTS_URLMAXSIZE * 2];
@@ -2182,12 +2178,13 @@ int httpmirror(char *url1, httrackp * opt) {
                         if (opt->log) {
                           hts_log_print(opt, LOG_INFO, "Purging directory %s/",
                                         file);
-                          while (strnotempty(file) &&
-                                 (hts_lastchar(file) != '/') &&
-                                 (hts_lastchar(file) != '\\')) {
-                            hts_choplastchar(file);
+                          while(strnotempty(file)
+                                && (file[strlen(file) - 1] != '/')
+                                && (file[strlen(file) - 1] != '\\')) {
+                            file[strlen(file) - 1] = '\0';
                           }
-                          hts_choplastchar(file);
+                          if (strnotempty(file))
+                            file[strlen(file) - 1] = '\0';
                         }
                       }
                     }
@@ -2289,7 +2286,6 @@ int httpmirror(char *url1, httrackp * opt) {
   usercommand(opt, 0, NULL, NULL, NULL, NULL);
   warc_close_opt(opt);
   hts_changes_close_opt(opt);
-  hts_sitemap_free(opt);
 
   // désallocation mémoire & buffers
   XH_uninit;
@@ -2638,11 +2634,7 @@ HTSEXT_API int structcheck(const char *path) {
       if (!S_ISDIR(st.st_mode)) {
 #if HTS_REMOVE_ANNOYING_INDEX
         if (S_ISREG(st.st_mode)) {      /* Regular file in place ; move it and create directory */
-          /* bounded here, not by the path-length guard far above */
-          if (!sprintfbuff(tmpbuf, "%s.txt", file)) {
-            errno = ENAMETOOLONG;
-            return -1;
-          }
+          sprintf(tmpbuf, "%s.txt", file);
           if (rename(file, tmpbuf) != 0) {      /* Can't rename regular file */
             return -1;
           }
@@ -2750,11 +2742,7 @@ HTSEXT_API int structcheck_utf8(const char *path) {
       if (!S_ISDIR(st.st_mode)) {
 #if HTS_REMOVE_ANNOYING_INDEX
         if (S_ISREG(st.st_mode)) {      /* Regular file in place ; move it and create directory */
-          /* bounded here, not by the path-length guard far above */
-          if (!sprintfbuff(tmpbuf, "%s.txt", file)) {
-            errno = ENAMETOOLONG;
-            return -1;
-          }
+          sprintf(tmpbuf, "%s.txt", file);
           if (RENAME(file, tmpbuf) != 0) {      /* Can't rename regular file */
             return -1;
           }
@@ -3148,15 +3136,16 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
   if (fp != NULL) {
     char s[256];
     time_t tt;
-    struct tm tmv;
+    struct tm *A;
 
     tt = time(NULL);
-    if (!hts_localtime(tt, &tmv)) {
+    A = localtime(&tt);
+    if (A == NULL) {
       int localtime_returned_null = 0;
 
       assertf(localtime_returned_null);
     }
-    strftime(s, 250, "%H:%M:%S", &tmv);
+    strftime(s, 250, "%H:%M:%S", A);
     if (strnotempty(type))
       fprintf(fp, "%s\t%c%s: \t", s, hichar(*type), type + 1);
     else
@@ -3695,10 +3684,6 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
   to->single_file = from->single_file;
   if (from->single_file_max_size > 0)
     to->single_file_max_size = from->single_file_max_size;
-  if (from->sitemap)
-    to->sitemap = from->sitemap;
-  if (StringNotEmpty(from->sitemap_url))
-    StringCopyS(to->sitemap_url, from->sitemap_url);
 
   if (from->pause_max_ms > 0) {
     to->pause_min_ms = from->pause_min_ms;
@@ -3773,11 +3758,11 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
         strcpybuff(codebase, heap(ptr)->fil);
       else
         strcpybuff(codebase, heap(heap(ptr)->precedent)->fil);
-      a = hts_lastcharptr(codebase);
+      a = codebase + strlen(codebase) - 1;
       while((*a) && (*a != '/') && (a > codebase))
         a--;
       if (*a == '/')
-        *(a + 1) = '\0';        // cut
+        *(a + 1) = '\0';        // couper
     } else {                    // couper http:// éventuel
       if (strfield(codebase, "http://")) {
         char BIGSTK tempo[HTS_URLMAXSIZE * 2];

src/htscore.h

@@ -376,8 +376,8 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
                           const char *template_footer, const char *adr,
                           const char *fil);
 
-// Flush ht_buff[0..ht_len] to save on disk; *fp closed+NULLed on write.
-// Precondition: ht_len>0.
+// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
+// closed+NULLed on write. Precondition: ht_len>0.
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save);

src/htscoremain.c

@@ -357,7 +357,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           char BIGSTK tempo[HTS_CDLMAXSIZE];
 
           strcpybuff(tempo, argv[na] + 1);
-          if (hts_lastchar(tempo) != '"') {
+          if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
             char BIGSTK s[HTS_CDLMAXSIZE];
 
             sprintf(s, "Missing quote in %s", argv[na]);
@@ -365,7 +365,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             htsmain_free();
             return -1;
           }
-          hts_choplastchar(tempo);
+          tempo[strlen(tempo) - 1] = '\0';
           /* tempo is argv[na] minus its surrounding quotes, so it fits in place
            */
           strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
@@ -863,7 +863,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         char BIGSTK tempo[HTS_CDLMAXSIZE + 256];
 
         strcpybuff(tempo, argv[na] + 1);
-        if (hts_lastchar(tempo) != '"') {
+        if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
           char s[HTS_CDLMAXSIZE + 256];
 
           sprintf(s, "Missing quote in %s", argv[na]);
@@ -871,7 +871,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           htsmain_free();
           return -1;
         }
-        hts_choplastchar(tempo);
+        tempo[strlen(tempo) - 1] = '\0';
         /* tempo is argv[na] minus its surrounding quotes, so it fits in place
          */
         strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
@@ -1833,26 +1833,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   }
                 }
                 break;
-              case 'm': // sitemap / sitemap-url: seed the crawl from sitemaps
-                if (*(com + 1) == 'u') { // --sitemap-url URL: explicit sitemap
-                  com++;
-                  if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                    HTS_PANIC_PRINTF(
-                        "Option sitemap-url needs a blank space and a URL");
-                    htsmain_free();
-                    return -1;
-                  }
-                  na++;
-                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
-                    HTS_PANIC_PRINTF("Sitemap URL too long");
-                    htsmain_free();
-                    return -1;
-                  }
-                  StringCopy(opt->sitemap_url, argv[na]);
-                } else { // --sitemap: robots.txt probe, then /sitemap.xml
-                  opt->sitemap = HTS_TRUE;
-                }
-                break;
               case 'Y': // why: explain the filter verdict for a URL, no crawl
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
@@ -2160,9 +2140,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               case 'R':
                 {
                   char *name;
-                  const char *why;
-                  unsigned long repaired = 0;
-                  unsigned long repairedBytes = 0;
+                  uLong repaired = 0;
+                  uLong repairedBytes = 0;
 
                   if (fexist_utf8(fconcat(
                           OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -2185,15 +2164,24 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     return 1;
                   }
                   fprintf(stderr, "Cache: trying to repair %s\n", name);
-                  why = cache_repair(opt, name, &repaired, &repairedBytes);
-                  if (why != NULL) {
-                    fprintf(stderr, "Cache: %s\n", why);
-                    return 1;
+                  if (unzRepair
+                      (name,
+                       fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                               "hts-cache/repair.zip"),
+                       fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                               "hts-cache/repair.tmp"), &repaired,
+                       &repairedBytes) == Z_OK) {
+                    UNLINK(name);
+                    RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                   StringBuff(opt->path_log),
+                                   "hts-cache/repair.zip"),
+                           name);
+                    fprintf(stderr,
+                            "Cache: %d bytes successfully recovered in %d entries\n",
+                            (int) repairedBytes, (int) repaired);
+                  } else {
+                    fprintf(stderr, "Cache: could not repair the cache\n");
                   }
-                  fprintf(
-                      stderr,
-                      "Cache: %d bytes successfully recovered in %d entries\n",
-                      (int) repairedBytes, (int) repaired);
                 }
                 return 0;
                 break;
@@ -2716,7 +2704,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       char *a;
 
       strcpybuff(rpath, StringBuff(opt->path_html));
-      hts_striplastchar(rpath, '/');
+      if (rpath[0]) {
+        if (rpath[strlen(rpath) - 1] == '/')
+          rpath[strlen(rpath) - 1] = '\0';
+      }
       a = strrchr(rpath, '/');
       if (a) {
         *a = '\0';

src/htsencoding.c

@@ -63,13 +63,6 @@ static int get_hex_value(char c) {
   } while (0)
 
 int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
-  return hts_unescapeEntitiesWithCharsetSpecial(src, dest, max, charset, 0);
-}
-
-int hts_unescapeEntitiesWithCharsetSpecial(const char *src, char *dest,
-                                           const size_t max,
-                                           const char *charset,
-                                           const int flags) {
   size_t i, j, ampStart, ampStartDest;
   int uc;
   int hex;
@@ -127,18 +120,9 @@ int hts_unescapeEntitiesWithCharsetSpecial(const char *src, char *dest,
             char buffer[32];
             len = 0;
             if ( ( ulen = hts_writeUTF8(uc, buffer, sizeof(buffer)) ) != 0) {
-              const hts_boolean urlQuery =
-                  (flags & UNESCAPE_ENTITIES_URL_QUERY) != 0;
               char *s;
               buffer[ulen] = '\0';
-              /* Strict for a query only: a substituted '?' must not pass for
-                 the code point the document wrote. */
-              if (urlQuery) {
-                s = hts_convertStringFromUTF8Strict(buffer, strlen(buffer),
-                                                    charset);
-              } else {
-                s = hts_convertStringFromUTF8(buffer, strlen(buffer), charset);
-              }
+              s = hts_convertStringFromUTF8(buffer, strlen(buffer), charset);
               if (s != NULL) {
                 const size_t sLen = strlen(s);
                 if (sLen < maxOut) {
@@ -146,18 +130,7 @@ int hts_unescapeEntitiesWithCharsetSpecial(const char *src, char *dest,
                   memcpy(&dest[ampStartDest], s, sLen);
                   len = sLen;
                 }
-                freet(s);
-              } else if (urlQuery) {
-                /* URL Standard: an unrepresentable code point is written
-                   %26%23<decimal>%3B rather than left as source text. */
-                char esc[32];
-                const int escLen =
-                    snprintf(esc, sizeof(esc), "%%26%%23%d%%3B", uc);
-
-                if (escLen > 0 && (size_t) escLen < maxOut) {
-                  memcpy(&dest[ampStartDest], esc, (size_t) escLen);
-                  len = (size_t) escLen;
-                }
+                free(s);
               }
             }
           }

src/htsencoding.h

@@ -69,29 +69,6 @@ extern int hts_unescapeEntitiesWithCharset(const char *src,
                                            char *dest, const size_t max,
                                            const char *charset);
 
-/**
- * Flags for hts_unescapeEntitiesWithCharsetSpecial(). Values stay distinct from
- * unescapeFlags above: both reach their function as a plain int.
- **/
-typedef enum unescapeEntitiesFlags {
-  /** The destination is a URL query string: write a reference the charset can
-      not represent as %26%23<decimal>%3B (URL Standard), instead of source text
-      whose '&' and '#' would re-split the query. **/
-  UNESCAPE_ENTITIES_URL_QUERY = 2
-} unescapeEntitiesFlags;
-
-/**
- * Unescape HTML entities into their charset equivalents, "flags" being a mask
- * of UNESCAPE_ENTITIES_XXX constants.
- * Note: source and destination MUST NOT be the same with a flag that may grow
- * the string (UNESCAPE_ENTITIES_URL_QUERY).
- * Returns 0 upon success, -1 upon overflow or error.
- **/
-extern int hts_unescapeEntitiesWithCharsetSpecial(const char *src, char *dest,
-                                                  const size_t max,
-                                                  const char *charset,
-                                                  const int flags);
-
 /**
  * Unescape an URL-encoded string. The implicit charset is UTF-8.
  * In case of UTF-8 decoding error inside URL-encoded characters, 

src/htsftp.c

@@ -149,61 +149,6 @@ void ftp_split_userpass(const char *src, const char *end, char *user,
   }
 }
 
-/* Build "<verb> <path>", quoting a path the server could not parse bare. */
-static void ftp_command(char *line, size_t line_size, const char *verb,
-                        const char *path) {
-  if (strchr(path, ' ') != NULL || strchr(path, '\"') != NULL ||
-      strchr(path, '\'') != NULL)
-    snprintf(line, line_size, "%s \"%s\"", verb, path);
-  else
-    snprintf(line, line_size, "%s %s", verb, path);
-}
-
-/* MDTM reply "213 YYYYMMDDHHMMSS[.frac]" (RFC 3659, UTC) into tm_time. */
-static hts_boolean ftp_parse_mdtm(const char *line, struct tm *tm_time) {
-  int year, mon, mday, hour, min, sec;
-
-  if (sscanf(line, "213 %4d%2d%2d%2d%2d%2d", &year, &mon, &mday, &hour, &min,
-             &sec) != 6)
-    return HTS_FALSE;
-  if (year < 1900 || mon < 1 || mon > 12 || mday < 1 || mday > 31 || hour < 0 ||
-      hour > 23 || min < 0 || min > 59 || sec < 0 || sec > 60)
-    return HTS_FALSE;
-  memset(tm_time, 0, sizeof(*tm_time));
-  tm_time->tm_year = year - 1900;
-  tm_time->tm_mon = mon - 1;
-  tm_time->tm_mday = mday;
-  tm_time->tm_hour = hour;
-  tm_time->tm_min = min;
-  tm_time->tm_sec = sec;
-  tm_time->tm_isdst = 0;
-  return HTS_TRUE;
-}
-
-/* Whether the local copy can still be a prefix of the remote file. FTP has no
-   conditional fetch, so an unprovable answer has to be no (#823). */
-static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back,
-                                  time_t remote_mtime) {
-  const time_t local_mtime = get_filetime(back->url_sav);
-
-  if (back->r.totalsize <= back->range_req_size) {
-    hts_log_print(opt, LOG_DEBUG,
-                  "FTP: not resuming %s%s, the local copy is not shorter",
-                  back->url_adr, back->url_fil);
-    return HTS_FALSE;
-  }
-  /* Equality, not ordering: a copy this code did not stamp carries a local
-     clock time newer than any MDTM, which "not newer" would wave through. */
-  if (remote_mtime == (time_t) -1 || local_mtime == (time_t) -1 ||
-      remote_mtime != local_mtime) {
-    hts_log_print(opt, LOG_DEBUG,
-                  "FTP: not resuming %s%s, the copy is not dated as the remote",
-                  back->url_adr, back->url_fil);
-    return HTS_FALSE;
-  }
-  return HTS_TRUE;
-}
-
 // la véritable fonction une fois lancées les routines thread/fork
 int run_launch_ftp(FTPDownloadStruct * pStruct) {
   lien_back *back = pStruct->pBack;
@@ -239,7 +184,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   strcpybuff(back->r.msg, "");
   back->r.statuscode = 0;
   back->r.size = 0;
-  back->r.lastmodified[0] = '\0'; // a retry must not stamp the previous MDTM
 
   // récupérer user et pass si présents, et sauter user:id@ dans adr
   real_adr = strchr(back->url_adr, ':');
@@ -514,7 +458,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               char catbuff[CATBUFF_SIZE];
               char *ua = unescape_http(catbuff, sizeof(catbuff), ftp_filename);
 
-              ftp_command(line, sizeof(line), "SIZE", ua);
+              if ((strchr(ua, ' '))
+                  || (strchr(ua, '\"'))
+                  || (strchr(ua, '\''))
+                ) {
+                snprintf(line, sizeof(line), "SIZE \"%s\"", ua);
+              } else {
+                snprintf(line, sizeof(line), "SIZE %s", ua);
+              }
 
               // SIZE?
               strcpybuff(back->info, "size");
@@ -523,8 +474,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               _CHECK_HALT_FTP;
               if (line[0] == '2') {     // SIZE compris, ALORS tester REST (sinon pas tester: cf probleme des txt.gz decompresses a la volee)
                 char *szstr = strchr(line, ' ');
-                time_t remote_mtime = (time_t) -1;
-                struct tm remote_tm;
 
                 if (szstr) {
                   LLint size = 0;
@@ -534,40 +483,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                     back->r.totalsize = size;
                   }
                 }
-
-                // MDTM?
-                ftp_command(line, sizeof(line), "MDTM", ua);
-                strcpybuff(back->info, "mdtm");
-                send_line(soc_ctl, line);
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (ftp_parse_mdtm(line, &remote_tm)) {
-                  char date[256];
-
-                  time_rfc822(date, &remote_tm);
-                  /* Stamp the mirror as the HTTP path does, so a later pass
-                     compares server-clock times instead of crossing clocks. */
-                  back->r.lastmodified[0] = '\0';
-                  strlncatbuff(back->r.lastmodified, date,
-                               sizeof(back->r.lastmodified),
-                               sizeof(back->r.lastmodified) - 1);
-                  remote_mtime = timegm(&remote_tm);
-                }
-
-                /* Only over a copy back_add() judged partial: on --update every
-                   mirrored file exists, and resuming a complete one splices the
-                   old body into the new (#798). */
-                if (back->range_req_size > 0 && (transfer_list == 0) &&
-                    ftp_may_resume(opt, back, remote_mtime)) {
+                // REST?
+                if (fexist(back->url_sav) && (transfer_list == 0)) {
                   strcpybuff(back->info, "rest");
-                  snprintf(line, sizeof(line), "REST " LLintP,
-                           (LLint) back->range_req_size);
+                  snprintf(line, sizeof(line), "REST " LLintP, (LLint) fsize(back->url_sav));
                   send_line(soc_ctl, line);
                   get_ftp_line(soc_ctl, line, sizeof(line), timeout);
                   _CHECK_HALT_FTP;
                   if ((line[0] == '3') || (line[0] == '2')) {   // ok
                     rest_understood = 1;
-                  } // else never mind
+                  }             // sinon tant pis 
                 }
               }                 // sinon tant pis 
             }
@@ -692,19 +617,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
         // Ok, connexion initiée
         //
         if (soc_dat != INVALID_SOCKET) {
-          if (rest_understood) { // REST sent and understood
+          if (rest_understood) {        // REST envoyée et comprise
             file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 0, 1,
                         0);
-            /* The bytes already on disk count too, or the completeness check
-               below rejects every resumed transfer (#798). */
-            back->r.size = back->range_req_size;
             back->r.fp = fileappend(&opt->state.strc, back->url_sav);
           } else {
             file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1,
                         0);
-            /* Every failure exit below would else leave the mirror truncated
-               (#771); the resume branch appends and needs no backup. */
-            back_refetch_backup(opt, back);
             back->r.fp = filecreate(&opt->state.strc, back->url_sav);
           }
           strcpybuff(back->info, "receiving");
@@ -767,10 +686,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               fclose(back->r.fp);
               back->r.fp = NULL;
             }
-            /* back_flush_output() stamps only r.is_write transfers, which FTP
-               never is; a partial needs the remote's date too (#823). */
-            if (strnotempty(back->r.lastmodified))
-              set_filetime_rfc822(back->url_sav, back->r.lastmodified);
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
             back->r.statuscode = STATUSCODE_INVALID;

src/htsglobal.h

@@ -138,11 +138,10 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DOSNAME 0
 #endif
 
-// zlib is mandatory: the cache is a zip and minizip calls it regardless
+// utiliser zlib?
 #ifndef HTS_USEZLIB
+// autoload
 #define HTS_USEZLIB 1
-#elif !HTS_USEZLIB
-#error HTS_USEZLIB=0 is not a supported configuration
 #endif
 
 // brotli and zstd content codings; off unless the build opted in (configure,

src/htshelp.c

@@ -83,8 +83,9 @@ void infomsg(const char *msg) {
               /* try the flag as-is, then strip a trailing N as the numeric-arg
                  placeholder (cN -> c); this order keeps -%N from becoming -% */
               p = optreal_find(cmd);
-              if (p < 0 && (int) strlen(cmd) > 2 && hts_lastchar(cmd) == 'N') {
-                hts_striplastchar(cmd, 'N');
+              if (p < 0 && (int) strlen(cmd) > 2 &&
+                  cmd[strlen(cmd) - 1] == 'N') {
+                cmd[strlen(cmd) - 1] = '\0';
                 p = optreal_find(cmd);
               }
               if (p >= 0) {
@@ -210,7 +211,7 @@ void help_wizard(httrackp * opt) {
     strcatbuff(str, "/websites/");
   }
   if (strnotempty(str))
-    if ((hts_lastchar(str) != '/') && (hts_lastchar(str) != '\\'))
+    if ((str[strlen(str) - 1] != '/') && (str[strlen(str) - 1] != '\\'))
       strcatbuff(str, "/");
   strcatbuff(stropt2, "-O \"");
   strcatbuff(stropt2, str);
@@ -525,11 +526,6 @@ void help(const char *app, int more) {
     (" %L <file> add all URL located in this text file (one URL per line)");
   infomsg
     (" %S <file> add all scan rules located in this text file (one scan rule per line)");
-  infomsg(" %m  seed the crawl from the site's sitemap (robots.txt Sitemap:, "
-          "then /sitemap.xml); --sitemap-url URL names one explicitly. A "
-          "sitemap you name, or one the site declares, is fetched even under "
-          "robots.txt Disallow; only the guessed /sitemap.xml obeys it. The "
-          "URLs found still pass every filter and scope rule");
   infomsg("");
   infomsg("Build options:");
   infomsg("  NN structure type (0 *original structure, 1+: see below)");
@@ -558,7 +554,8 @@ void help(const char *app, int more) {
   infomsg
     (" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
   infomsg(" %g  strip query keys for dedup ([host/pattern=]key1,key2,...)");
-  infomsg("  o *save the server's error pages (404..) (o0 discard them)");
+  infomsg
+    ("  o *generate output html file in case of error (404..) (o0 don't generate)");
   infomsg("  X *purge old files after update (X0 keep delete)");
   infomsg(" %p  preserve html files 'as is' (identical to '-K4 -%F \"\"')");
   infomsg(" %T  links conversion to UTF-8");

src/htslib.c

@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com
 // Fichier librairie .c
 
 #include "htscore.h"
-#include "htssitemap.h"
 #include "htswarc.h"
 #include "htschanges.h"
 #include "htssinglefile.h"
@@ -1131,10 +1130,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
 
       // Compression accepted ?
       if (retour->req.http11) {
-        hts_boolean compressible =
-            (!retour->req.range_used && !retour->req.nocompression);
+        hts_boolean compressible = HTS_FALSE;
         hts_boolean secure = HTS_FALSE;
 
+#if HTS_USEZLIB
+        compressible = (!retour->req.range_used && !retour->req.nocompression);
+#endif
 #if HTS_USEOPENSSL
         secure = retour->ssl ? HTS_TRUE : HTS_FALSE;
 #endif
@@ -1322,7 +1323,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       p++;                      // sauter espaces
     if ((int) strlen(rcvd + p) < 250) { // pas trop long?
       char tmp[256];
-      char *a = NULL;
+      char *a = NULL, *b = NULL;
 
       strcpybuff(tmp, rcvd + p);
       a = strstr(tmp, "filename=");
@@ -1335,9 +1336,15 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
 
           while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
             a = c + 1;
-          hts_rtrim(a, HTS_SPACES);
-          if ((int) strlen(a) < 200) { // pas trop long?
-            strcpybuff(retour->cdispo, a);
+          b = a + strlen(a) - 1;
+          while(is_space(*b))
+            b--;
+          b++;
+          if (b) {
+            *b = '\0';
+            if ((int) strlen(a) < 200) {        // pas trop long?
+              strcpybuff(retour->cdispo, a);
+            }
           }
         }
       }
@@ -2684,36 +2691,41 @@ HTSEXT_API void qsec2str(char *st, TStamp t) {
 // heure actuelle, GMT, format rfc (taille buffer 256o)
 void time_gmt_rfc822(char *s) {
   time_t tt;
-  struct tm tmv;
+  struct tm *A;
 
   tt = time(NULL);
-  /* no local-time fallback: it would format local time and still label it GMT
-   * (#806) */
-  if (hts_gmtime(tt, &tmv))
-    time_rfc822(s, &tmv);
-  else
-    s[0] = '\0';
+  A = gmtime(&tt);
+  if (A == NULL)
+    A = localtime(&tt);
+  time_rfc822(s, A);
 }
 
 void hts_now_iso8601(char out[32]) {
   time_t t = time(NULL);
   struct tm tmv;
 
-  if (!hts_gmtime(t, &tmv))
+#if defined(_WIN32)
+  struct tm *g = gmtime(&t);
+
+  if (g != NULL)
+    tmv = *g;
+  else
     memset(&tmv, 0, sizeof(tmv));
+#else
+  if (gmtime_r(&t, &tmv) == NULL)
+    memset(&tmv, 0, sizeof(tmv));
+#endif
   strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &tmv);
 }
 
 // heure actuelle, format rfc (taille buffer 256o)
 void time_local_rfc822(char *s) {
   time_t tt;
-  struct tm tmv;
+  struct tm *A;
 
   tt = time(NULL);
-  if (hts_localtime(tt, &tmv))
-    time_rfc822_local(s, &tmv);
-  else
-    s[0] = '\0';
+  A = localtime(&tt);
+  time_rfc822_local(s, A);
 }
 
 /* convertir une chaine en temps */
@@ -2846,28 +2858,20 @@ int set_filetime_rfc822(const char *file, const char *date) {
     return -1;
 }
 
-/* Note: utf-8 */
-time_t get_filetime(const char *file) {
-  STRUCT_STAT buf;
-
-  if (STAT(file, &buf) != 0)
-    return (time_t) -1;
-  return (time_t) buf.st_mtime;
-}
-
 /* Note: utf-8 */
 int get_filetime_rfc822(const char *file, char *date) {
   STRUCT_STAT buf;
 
   date[0] = '\0';
   if (STAT(file, &buf) == 0) {
-    struct tm tmv;
+    struct tm *A;
     time_t tt = buf.st_mtime;
 
-    /* no local-time fallback: it would format local time and still label it GMT
-     * (#806) */
-    if (hts_gmtime(tt, &tmv)) {
-      time_rfc822(date, &tmv);
+    A = gmtime(&tt);
+    if (A == NULL)
+      A = localtime(&tt);
+    if (A != NULL) {
+      time_rfc822(date, A);
       return 1;
     }
   }
@@ -3312,9 +3316,8 @@ int ishtml(httrackp * opt, const char *fil) {
   }
 
   /* Search for known ext */
-  for (a = hts_lastcharptr(fil_noquery);
-       *a != '.' && *a != '/' && a > fil_noquery; a--)
-    ;
+  for(a = fil_noquery + strlen(fil_noquery) - 1;
+      *a != '.' && *a != '/' && a > fil_noquery; a--) ;
   if (*a == '.') {              // a une extension
     char BIGSTK fil_noquery[HTS_URLMAXSIZE * 2];
     char *b;
@@ -4251,8 +4254,9 @@ HTSEXT_API hts_boolean get_httptype_sized(httrackp *opt, char *s, size_t ssize,
     return 1;
   } else {
     /* Check html -> text/html */
-    const char *a = hts_lastcharptr(fil);
+    const char *a = fil + strlen(fil) - 1;
 
+    /* a < fil when fil is empty: bound before dereferencing */
     while ((a > fil) && (*a != '.') && (*a != '/'))
       a--;
     if (a >= fil && *a == '.' && strlen(a) < 32) {
@@ -5075,7 +5079,7 @@ static int hts_dns_resolve_nocache_list(const char *const hostname,
   if (!strnotempty(hostname) || max <= 0) {
     return 0;
   }
-  if ((hostname[0] == '[') && (hts_lastchar(hostname) == ']')) {
+  if ((hostname[0] == '[') && (hostname[strlen(hostname) - 1] == ']')) {
     size_t size = strlen(hostname);
     char *copy = malloct(size + 1);
     int count;
@@ -5488,8 +5492,9 @@ void cut_path(char *fullpath, char *path, size_t path_size, char *pname,
               size_t pname_size) {
   path[0] = pname[0] = '\0';
   if (strnotempty(fullpath)) {
-    if (!hts_striplastchar(fullpath, '/'))
-      hts_striplastchar(fullpath, '\\');
+    if ((fullpath[strlen(fullpath) - 1] == '/')
+        || (fullpath[strlen(fullpath) - 1] == '\\'))
+      fullpath[strlen(fullpath) - 1] = '\0';
     if (strlen(fullpath) > 1) {
       char *a;
 
@@ -5787,8 +5792,7 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_
   if (hts_log_print_callback != NULL) {
     va_list args_copy;
     va_copy(args_copy, args);
-    /* the copy, so the vfprintf() below still has an unread list */
-    hts_log_print_callback(opt, type, format, args_copy);
+    hts_log_print_callback(opt, type, format, args);
     va_end(args_copy);
   }
   if (opt != NULL && opt->log != NULL) {
@@ -6026,7 +6030,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
   StringCopy(opt->warc_file, "");
-  StringCopy(opt->sitemap_url, "");
   opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
   opt->changes = HTS_FALSE;
   opt->changes_state = NULL;
@@ -6184,8 +6187,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->cookies_file);
     StringFree(opt->why_url);
     StringFree(opt->warc_file);
-    StringFree(opt->sitemap_url);
-    hts_sitemap_free(opt); /* backstop: httpmirror's early-return paths */
 
     hts_changes_free_opt(opt);
 
@@ -6645,12 +6646,9 @@ int hts_rename_utf8(const char *oldpath, const char *newpath) {
   LPWSTR wnewpath = hts_pathToUCS2(newpath);
   if (woldpath != NULL && wnewpath != NULL) {
     const int result = _wrename(woldpath, wnewpath);
-    /* Save errno: callers key off it (#779) and free() may clobber it. */
-    const int err = errno;
 
     free(woldpath);
     free(wnewpath);
-    errno = err;
     return result;
   } else {
     if (woldpath != NULL)

src/htslib.h

@@ -157,30 +157,6 @@ struct t_dnscache {
   char host_addr[HTS_MAXADDRNUM][HTS_MAXADDRLEN];
 };
 
-/* Break t down as UTC into the caller's buffer, HTS_FALSE if that failed.
-   gmtime()'s static is shared, and both the engine and ProxyTrack convert on
-   worker threads. */
-static HTS_INLINE HTS_UNUSED hts_boolean hts_gmtime(time_t t,
-                                                    struct tm *tmbuf) {
-#ifdef _WIN32
-  /* Microsoft's gmtime_s takes the destination first, unlike C11 Annex K. */
-  return gmtime_s(tmbuf, &t) == 0 ? HTS_TRUE : HTS_FALSE;
-#else
-  return gmtime_r(&t, tmbuf) != NULL ? HTS_TRUE : HTS_FALSE;
-#endif
-}
-
-/* Break t down as local time into the caller's buffer, HTS_FALSE if that
-   failed. localtime()'s static is shared, same rationale as hts_gmtime(). */
-static HTS_INLINE HTS_UNUSED hts_boolean hts_localtime(time_t t,
-                                                       struct tm *tmbuf) {
-#ifdef _WIN32
-  return localtime_s(tmbuf, &t) == 0 ? HTS_TRUE : HTS_FALSE;
-#else
-  return localtime_r(&t, tmbuf) != NULL ? HTS_TRUE : HTS_FALSE;
-#endif
-}
-
 /* Library internal definictions */
 #ifdef HTS_INTERNAL_BYTECODE
 
@@ -289,8 +265,6 @@ void hts_now_iso8601(char out[32]);
 struct tm *convert_time_rfc822(struct tm *buffer, const char *s);
 int set_filetime(const char *file, struct tm *tm_time);
 int set_filetime_rfc822(const char *file, const char *date);
-/* File mtime as a UTC time_t, or (time_t) -1 if it can not be read. */
-time_t get_filetime(const char *file);
 int get_filetime_rfc822(const char *file, char *date);
 HTS_INLINE void time_rfc822(char *s, struct tm *A);
 HTS_INLINE void time_rfc822_local(char *s, struct tm *A);

src/htsname.c

@@ -43,7 +43,9 @@ Please visit our Website: http://www.httrack.com
 #include "htsencoding.h"
 #include "htssniff.h"
 #include "htscodec.h"
+#if HTS_USEZLIB
 #include "htszlib.h"
+#endif
 #include <ctype.h>
 #include <limits.h>
 
@@ -65,76 +67,14 @@ Please visit our Website: http://www.httrack.com
 
 /* Avoid stupid DOS system folders/file such as 'nul' */
 /* Based on linux/fs/umsdos/mangle.c */
-/* Hand-grouped rows; clang-format reflows the table into a blob. */
-/* clang-format off */
 static const char *hts_tbdev[] = {
-  "prn", "con", "aux", "nul",
-  "lpt1", "lpt2", "lpt3", "lpt4",
-  "com1", "com2", "com3", "com4",
-  "clock$",
-  "emmxxxx0", "xmsxxxx0", "setverxx",
+  "/prn", "/con", "/aux", "/nul",
+  "/lpt1", "/lpt2", "/lpt3", "/lpt4",
+  "/com1", "/com2", "/com3", "/com4",
+  "/clock$",
+  "/emmxxxx0", "/xmsxxxx0", "/setverxx",
   ""
 };
-/* clang-format on */
-
-/* Directories the engine owns inside the mirror: a URL naming one lands on the
-   cache and destroys it (#774). Defence in depth; the temporaries themselves
-   live where no savename can spell them (HTS_TMPDIR in htsback.c). */
-static const char *hts_tbreserved[] = {"hts-cache", "hts-tmp", ""};
-
-/* True once the component holds only what cleanEndingSpaceOrDot() strips. */
-static hts_boolean strippedToComponentEnd(const char *s) {
-  while (*s == ' ' || *s == '.')
-    s++;
-  return *s == '\0' || *s == '/' ? HTS_TRUE : HTS_FALSE;
-}
-
-/* Replace foo/<reserved>/bar by foo/<reserved>_/bar, matching a whole path
-   component only (case-insensitively: the filesystem may be too). */
-static void escapeReservedNames(char *save, size_t size,
-                                const char *const *names) {
-  int i;
-
-  for (i = 0; names[i][0] != '\0'; i++) {
-    const char *a = save;
-    const size_t len = strlen(names[i]);
-
-    while ((a = strstrcase(a, names[i]))) {
-      hts_boolean reserved = HTS_FALSE;
-
-      if (a == save) {
-        /* save has had its leading '/' stripped above, so the table's anchor
-           never reached the first component (#842). It usually holds the
-           hostname, so only a trailing run may end it here: '.' would rename
-           aux.example.com. */
-        reserved = strippedToComponentEnd(a + len);
-      } else if (a[-1] == '/') {
-        switch ((int) a[len]) {
-        case '\0':
-        case '/':
-        case '.':
-          reserved = HTS_TRUE;
-          break;
-        case ' ':
-          /* cleanEndingSpaceOrDot() runs after us and hands the name back */
-          reserved = strippedToComponentEnd(a + len);
-          break;
-        }
-      }
-      if (reserved) {
-        char BIGSTK tempo[HTS_URLMAXSIZE * 2];
-
-        tempo[0] = '\0';
-        strncatbuff(tempo, save, (int) (a - save) + (int) len);
-        strcatbuff(tempo, "_");
-        strcatbuff(tempo, a + len);
-        /* clip rather than abort: the name comes from the wire */
-        (void) strclipbuff(save, size, tempo);
-      }
-      a += len;
-    }
-  }
-}
 
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
@@ -499,7 +439,9 @@ int url_savename(lien_adrfilsave *const afs,
 
       strcpybuff(fil_complete_patche, normfil);
       // Version avec ou sans /
-      if (!hts_striplastchar(fil_complete_patche, '/'))
+      if (fil_complete_patche[strlen(fil_complete_patche) - 1] == '/')
+        fil_complete_patche[strlen(fil_complete_patche) - 1] = '\0';
+      else
         strcatbuff(fil_complete_patche, "/");
       i = hash_read(hash, normadr, fil_complete_patche, HASH_STRUCT_ORIGINAL_ADR_PATH);       // recherche table 2 (former->adr+former->fil)
       if (i >= 0) {
@@ -575,8 +517,7 @@ int url_savename(lien_adrfilsave *const afs,
         && protocol != PROTOCOL_FTP
       ) {
       // tester type avec requète HEAD si on ne connait pas le type du fichier
-      if (!((opt->check_type == 1) &&
-            (hts_lastchar(fil) == '/'))) // slash doit être html?
+      if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/')))   // slash doit être html?
         if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
             ishtml(opt, fil) < 0) { // unsure whether it's html or a file
           // lire dans le cache
@@ -871,7 +812,7 @@ int url_savename(lien_adrfilsave *const afs,
   // - - - DEBUT NOMMAGE - - -
 
   // Donner nom par défaut?
-  if (hts_lastchar(fil) == '/') {
+  if (fil[strlen(fil) - 1] == '/') {
     if (!strfield(adr_complete, "ftp://")
       ) {
       strcatbuff(fil, DEFAULT_HTML);    // nommer page par défaut!!
@@ -885,7 +826,7 @@ int url_savename(lien_adrfilsave *const afs,
   // Change the extension? e.g. php3 saved as html, cgi as html or gif/xbm
   // depending on the resolved type.
   if (ext_chg && !opt->no_type_change) {
-    char *a = hts_lastcharptr(fil);
+    char *a = fil + strlen(fil) - 1;
 
     if ((opt->debug > 1) && (opt->log != NULL)) {
       if (ext_chg == 1)
@@ -920,7 +861,7 @@ int url_savename(lien_adrfilsave *const afs,
   }
   // Rechercher premier / et dernier .
   {
-    const char *a = hts_lastcharptr(fil);
+    const char *a = fil + strlen(fil) - 1;
 
     // passer structures
     start_pos = fil;
@@ -1265,29 +1206,29 @@ int url_savename(lien_adrfilsave *const afs,
     switch (opt->savename_type % 100) {
     case 4:
     case 5:{                   // séparer par types
-      const char *a = hts_lastcharptr(fil);
+        const char *a = fil + strlen(fil) - 1;
 
-      // passer structures
-      while ((a > fil) && (*a != '/') && (*a != '\\'))
-        a--;
-      if ((*a == '/') || (*a == '\\'))
-        a++;
-
-      // html?
-      if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) {
-        if (opt->savename_type % 100 == 5)
-          strcatbuff(afs->save, "html/");
-      } else {
-        const char *a = hts_lastcharptr(fil);
-
-        while ((a > fil) && (*a != '/') && (*a != '.'))
+        // passer structures
+        while((a > fil) && (*a != '/') && (*a != '\\'))
           a--;
-        if (*a != '.')
-          strcatbuff(afs->save, "other");
-        else
-          strcatbuff(afs->save, a + 1);
-        strcatbuff(afs->save, "/");
-      }
+        if ((*a == '/') || (*a == '\\'))
+          a++;
+
+        // html?
+        if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) {
+          if (opt->savename_type % 100 == 5)
+            strcatbuff(afs->save, "html/");
+        } else {
+          const char *a = fil + strlen(fil) - 1;
+
+          while((a > fil) && (*a != '/') && (*a != '.'))
+            a--;
+          if (*a != '.')
+            strcatbuff(afs->save, "other");
+          else
+            strcatbuff(afs->save, a + 1);
+          strcatbuff(afs->save, "/");
+        }
         /*strcatbuff(save,a); */
         /* add name */
         ADD_STANDARD_NAME(0);
@@ -1320,7 +1261,7 @@ int url_savename(lien_adrfilsave *const afs,
         }
         afs->save[i + j] = '\0';
         // ajouter extension
-        a = hts_lastcharptr(fil);
+        a = fil + strlen(fil) - 1;
         while((a > fil) && (*a != '/') && (*a != '.'))
           a--;
         if (*a == '.') {
@@ -1345,7 +1286,7 @@ int url_savename(lien_adrfilsave *const afs,
 
     hts_lowcase(afs->save);
 
-    if (hts_lastchar(afs->save) == '/')
+    if (afs->save[strlen(afs->save) - 1] == '/')
       strcatbuff(afs->save, DEFAULT_HTML);   // nommer page par défaut!!
   }
 
@@ -1365,7 +1306,7 @@ int url_savename(lien_adrfilsave *const afs,
   // cela évite les /chez/toto et les /chez/toto/index.html incompatibles
   if (opt->savename_type != -1 &&
       opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD) {
-    char *a = hts_lastcharptr(afs->save);
+    char *a = afs->save + strlen(afs->save) - 1;
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
@@ -1431,12 +1372,35 @@ int url_savename(lien_adrfilsave *const afs,
   // éliminer les // (comme ftp://)
   cleanDoubleSlash(afs->save);
 
-  /* Runs on every platform, and before path_html is prepended below, so the
-     user's own output directory is never renamed. */
-  escapeReservedNames(afs->save, sizeof(afs->save), hts_tbreserved);
-
 #if HTS_OVERRIDE_DOS_FOLDERS
-  escapeReservedNames(afs->save, sizeof(afs->save), hts_tbdev);
+  /* Replace /foo/nul/bar by /foo/nul_/bar */
+  {
+    int i = 0;
+
+    while(hts_tbdev[i][0]) {
+      const char *a = afs->save;
+
+      while((a = strstrcase(a, hts_tbdev[i]))) {
+        switch ((int) a[strlen(hts_tbdev[i])]) {
+        case '\0':
+        case '/':
+        case '.':
+          {
+            char BIGSTK tempo[HTS_URLMAXSIZE * 2];
+
+            tempo[0] = '\0';
+            strncatbuff(tempo, afs->save, (int) (a - afs->save) + strlen(hts_tbdev[i]));
+            strcatbuff(tempo, "_");
+            strcatbuff(tempo, a + strlen(hts_tbdev[i]));
+            strcpybuff(afs->save, tempo);
+          }
+          break;
+        }
+        a += strlen(hts_tbdev[i]);
+      }
+      i++;
+    }
+  }
 
   /* Strip ending . or ' ' forbidden on windoz */
   cleanEndingSpaceOrDot(afs->save);
@@ -1459,10 +1423,8 @@ int url_savename(lien_adrfilsave *const afs,
   if (opt->savename_83 > 0) {
     char *a, *last;
 
-    for (last = hts_lastcharptr(afs->save);
-         last != afs->save && *last != '/' && *last != '\\' && *last != '.';
-         last--)
-      ;
+    for(last = afs->save + strlen(afs->save) - 1;
+        last != afs->save && *last != '/' && *last != '\\' && *last != '.'; last--) ;
     if (*last != '.') {
       last = NULL;
     }
@@ -1545,7 +1507,6 @@ int url_savename(lien_adrfilsave *const afs,
 #endif
 #define MIN_LAST_SEG_RESERVE 12
 #define MAX_LAST_SEG_RESERVE 24
-#define MAX_EXT_LEN 12 /* longest tail kept across a cut, sans the dot */
   if (hts_stringLengthUTF8(afs->save) +
       hts_stringLengthUTF8(StringBuff(opt->path_html_utf8)) >=
       HTS_MAX_PATH_LEN) {
@@ -1614,26 +1575,9 @@ int url_savename(lien_adrfilsave *const afs,
           markStart = (p > lastSeg && p < extDot && wsave[p - 1] == '.')
                           ? p - 1
                           : extDot;
-        } else {
-          // #852: reserve a plain extension too, or the cut costs the page
-          // the ".html" the mirror is browsed by.
-          size_t p = wsaveLen;
-
-          while (p > lastSeg && wsave[p - 1] != '.')
-            p--;
-          if (p > lastSeg + 1 && wsaveLen - p <= MAX_EXT_LEN) {
-            markStart = p - 1;
-          }
         }
-        // #852: clamp the name like any directory segment; it was bounded by
-        // the whole path alone, so one component could run to maxLen.
-        const size_t tailLen = wsaveLen - markStart;
-        const size_t headLen =
-            tailLen < MAX_SEG_LEN ? MAX_SEG_LEN - tailLen : 0;
-
-        // head, bounded so the reserved tail still fits, then the tail itself
-        for (i = lastSeg; i < markStart && i - lastSeg < headLen &&
-                          j < maxLen && tailLen < maxLen - j;
+        // head, bounded so the marker still fits, then the marker itself
+        for (i = lastSeg; i < markStart && j + (wsaveLen - markStart) < maxLen;
              i++)
           wsave[j++] = wsave[i];
         for (i = markStart; i < wsaveLen && j < maxLen; i++)
@@ -1686,7 +1630,6 @@ int url_savename(lien_adrfilsave *const afs,
 #undef MAX_UTF8_SEQ_CHARS
 #undef MIN_LAST_SEG_RESERVE
 #undef MAX_LAST_SEG_RESERVE
-#undef MAX_EXT_LEN
 #undef MAX_SEG_LEN
 #undef HTS_MAX_PATH_LEN
 #undef HTS_PATH_TAIL_RESERVE
@@ -1736,8 +1679,8 @@ int url_savename(lien_adrfilsave *const afs,
 #endif
         } else {                // utilisé par un AUTRE, changer de nom
           char BIGSTK tempo[HTS_URLMAXSIZE * 2];
-          char *a = hts_lastcharptr(afs->save);
-          size_t stem;
+          char *a = afs->save + strlen(afs->save) - 1;
+          char *b;
           int n = 2;
           char collisionSeparator =
               ((opt->savename_83 != HTS_SAVENAME_83_ISO9660) ? '-' : '_');
@@ -1759,16 +1702,18 @@ int url_savename(lien_adrfilsave *const afs,
             strcatbuff(tempo, afs->save);
 
           // tester la présence d'un -xx (ex: index-2.html -> index-3.html)
-          stem = hts_rtrimlen(tempo, "0123456789");
-          if (stem != 0 && tempo[stem - 1] == collisionSeparator) {
-            sscanf(tempo + stem, "%d", &n);
-            tempo[stem - 1] = '\0'; // couper
+          b = tempo + strlen(tempo) - 1;
+          while(isdigit((unsigned char) *b))
+            b--;
+          if (*b == collisionSeparator) {
+            sscanf(b + 1, "%d", &n);
+            *b = '\0';          // couper
             n++;                // plus un
           }
           // en plus il faut gérer le 8-3 .. pas facile le client
           if (opt->savename_83) {
             int max;
-            char *a = hts_lastcharptr(tempo);
+            char *a = tempo + strlen(tempo) - 1;
 
             while((a > tempo) && (*a != '/'))
               a--;

src/htsopt.h

@@ -557,13 +557,6 @@ struct httrackp {
   LLint single_file_max_size; /**< --single-file-max-size: per-asset cap in
                                    bytes; a bigger asset stays a link.
                                    Tail: ABI */
-  hts_boolean sitemap; /**< --sitemap: probe the start host's robots.txt for
-                            Sitemap: lines, else /sitemap.xml. Tail: ABI */
-  String sitemap_url;  /**< --sitemap-url: sitemap to ingest. Tail: ABI */
-  /* Live state, not an option: copy_htsopt must leave it alone. It sits here
-     rather than in htsoptstate because that struct is embedded by value, so
-     growing it would shift every httrackp field declared after it. */
-  void *sitemap_state; /**< hts_sitemap_state*, or NULL. Tail: ABI */
 };
 
 /* Running statistics for a mirror. */
@@ -695,9 +688,6 @@ struct htsblk {
   char *warc_rawpath; /**< verbatim WARC: spooled compressed body path, or NULL
                            (owns the file; unlinked on free). Tail: ABI */
   LLint warc_rawsize; /**< byte length of warc_rawpath. Tail: ABI */
-  /** notmodified came from an engine hack, not a server 304 (#839).
-      Tail: ABI */
-  hts_boolean warc_forced_notmodified;
   /*char digest[32+2];   // md5 digest generated by the engine ("" if none) */
 };
 

src/htsparse.c

@@ -59,6 +59,7 @@ Please visit our Website: http://www.httrack.com
 
 // parser
 #include "htsparse.h"
+#include "htssinglefile.h"
 #include "htsback.h"
 
 // arrays
@@ -279,20 +280,6 @@ static void url_drop_fragment(char *const url) {
     *frag = '\0';
 }
 
-/* Charset to decode the page with, or NULL when conversion is off. *declared
-   tells a charset the document named from the iso-8859-1 guess standing in. */
-static const char *page_charset(const htsmoduleStruct *str,
-                                hts_boolean *declared) {
-  const char *const charset = str->page_charset_;
-
-  if (charset == NULL) {
-    *declared = HTS_FALSE;
-    return NULL;
-  }
-  *declared = *charset != '\0' ? HTS_TRUE : HTS_FALSE;
-  return *declared ? charset : "iso-8859-1";
-}
-
 /* True if [s, s+len) is exactly an HTTP method token (XHR.open's first
    argument is a method, not a URL: #218). Case-insensitive. */
 static int is_http_method(const char *s, size_t len) {
@@ -699,22 +686,14 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         }
 
                         // Decode title with encoding
-                        {
-                          hts_boolean declared;
-                          const char *const charset =
-                              page_charset(str, &declared);
-
-                          // Guessed charset: never re-encode valid UTF-8 (#833)
-                          if (charset != NULL && !hts_isCharsetUTF8(charset) &&
-                              (declared || !hts_isStringUTF8(s, strlen(s)))) {
-                            char *sUtf =
-                                hts_convertStringToUTF8(s, strlen(s), charset);
-
-                            if (sUtf != NULL) {
-                              /* UTF-8 can expand past s[]; truncate to fit */
-                              snprintf(s, sizeof(s), "%s", sUtf);
-                              freet(sUtf);
-                            }
+                        if (str->page_charset_ != NULL &&
+                            *str->page_charset_ != '\0') {
+                          char *sUtf = hts_convertStringToUTF8(
+                              s, strlen(s), str->page_charset_);
+                          if (sUtf != NULL) {
+                            /* UTF-8 can expand past s[]; truncate to fit */
+                            snprintf(s, sizeof(s), "%s", sUtf);
+                            freet(sUtf);
                           }
                         }
 
@@ -1723,8 +1702,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 #endif
                                     )   // ok pas de problème
                                     url_ok = 1;
-                                  else if (hts_lastchar(tempo) ==
-                                           '/') {       // un slash: ok..
+                                  else if (tempo[strlen(tempo) - 1] == '/') {   // un slash: ok..
                                     if (inscript)       // sinon si pas javascript, méfiance (répertoire style base?)
                                       url_ok = 1;
                                   }
@@ -2028,8 +2006,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               if (eadr - html - 1 < HTS_URLMAXSIZE) {        // pas trop long?
                 strncpy(lien, html, eadr - html - 1);
                 lien[eadr - html - 1] = '\0';
-                while (hts_striplastchar(lien, ' ')) {
-                }
+                // supprimer les espaces
+                while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
+                  lien[strlen(lien) - 1] = '\0';
 
               } else
                 lien[0] = '\0'; // erreur
@@ -2079,7 +2058,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   }
                   q = strchr(a, '?');   // ne pas traiter après '?'
                   if (!q)
-                    q = hts_lastcharptr(a);
+                    q = a + strlen(a) - 1;
                   while((p = strstr(a, "//")) && (!done)) {     // remplacer // par /
                     if (p > q) {    // après le ? (toto.cgi?param=1//2.3)
                       done = 1; // stopper
@@ -2132,9 +2111,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 /* Unescape/escape %20 and other &nbsp; */
                 {
                   // NULL when UTF-8 conversion is off (-%T0)
-                  hts_boolean declared;
-                  const char *const charset = page_charset(str, &declared);
-                  const int hasCharset = charset != NULL;
+                  const char *const charset = str->page_charset_;
+                  const int hasCharset = charset != NULL 
+                    && *charset != '\0';
                   char BIGSTK query[HTS_URLMAXSIZE * 2];
 
                   // cut query string
@@ -2176,20 +2155,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       "could not decode URI '%s' with charset '%s'", lien, charset);
                   }
 
-                  // decode query entities with the page charset, out-of-place
-                  // because the escape grows it (#854)
+                  // decode query string entities with page charset
                   if (hasCharset) {
-                    char BIGSTK decoded[sizeof(query)];
-
-                    if (hts_unescapeEntitiesWithCharsetSpecial(
-                            query, decoded, sizeof(decoded), charset,
-                            UNESCAPE_ENTITIES_URL_QUERY) == 0) {
-                      strcpybuff(query, decoded);
-                    } else {
-                      hts_log_print(opt, LOG_WARNING,
-                                    "could not decode query string '%s' with "
-                                    "charset '%s'",
-                                    query, charset);
+                    if (hts_unescapeEntitiesWithCharset(query, 
+                                                        query, strlen(query) + 1,
+                                                        charset) != 0) {
+                        hts_log_print(opt, LOG_WARNING,
+                          "could not decode query string '%s' with charset '%s'", query, charset);
                     }
                   }
 
@@ -2236,8 +2208,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 // supposition dangereuse?
                 // OUI!!
 #if HTS_TILDE_SLASH
-                if (hts_lastchar(lien) != '/') {
-                  char *a = hts_lastcharptr(lien);
+                if (lien[strlen(lien) - 1] != '/') {
+                  char *a = lien + strlen(lien) - 1;
 
                   // éviter aussi index~1.html
                   while(a > lien && (*a != '~') && (*a != '/')
@@ -2283,7 +2255,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 // vérifier que l'on ne doit pas ajouter de .class
                 if (!error) {
                   if (add_class) {
-                    char *a = hts_lastcharptr(lien);
+                    char *a = lien + strlen(lien) - 1;
 
                     while((a > lien) && (*a != '/') && (*a != '.'))
                       a--;
@@ -2301,7 +2273,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     // Vérifier les codebase=applet (au lieu de applet/)
                     if (p_type == -2) { // codebase
                       if (strnotempty(lien)) {
-                        if (hts_lastchar(lien) != '/') { // pas répertoire
+                        if (lien[strlen(lien) - 1] != '/') {     // pas répertoire
                           strcatbuff(lien, "/");
                         }
                       }
@@ -2338,7 +2310,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         {
                           char *a;
 
-                          a = hts_lastcharptr(lien);
+                          a = lien + strlen(lien) - 1;
                           while((*a) && (*a != '/') && (a > lien))
                             a--;
                           if (*a == '/') {
@@ -2349,8 +2321,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     }
 
                     if (!error) {       // pas d'erreur?
-                      if (p_type == 2) { // code ET PAS codebase
-                        char *a = hts_lastcharptr(lien);
+                      if (p_type == 2) {        // code ET PAS codebase      
+                        char *a = lien + strlen(lien) - 1;
                         char *start_of_filename = jump_identification(lien);
 
                         if (start_of_filename != NULL
@@ -2717,11 +2689,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             int cat_data_len = 0;
 
                             // ajouter lien external
-                            switch ((link_has_authority(afs.af.adr))
-                                        ? 1
-                                        : ((hts_lastchar(afs.af.fil) == '/')
-                                               ? 1
-                                               : (ishtml(opt, afs.af.fil)))) {
+                            switch ((link_has_authority(afs.af.adr)) ? 1
+                                    : ((afs.af.fil[strlen(afs.af.fil) - 1] ==
+                                        '/') ? 1 : (ishtml(opt, afs.af.fil)))) {
                             case 1:
                             case -2:   // html ou répertoire
                               if (opt->getmode & HTS_GETMODE_HTML) {
@@ -2764,7 +2734,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                 cat_data_len = HTS_DATA_UNKNOWN_HTML_LEN;
                               }
                               break;
-                            } // html,gif
+                            }   // html,gif
 
                             if (patch_it) {
                               char BIGSTK save[HTS_URLMAXSIZE * 2];
@@ -3074,6 +3044,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                         if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
 
+                          /* --single-file: tag the reference for the
+                             end-of-mirror pass. The tag is a fragment, so the
+                             mirror stays valid if that pass never runs. */
+                          if (opt->single_file && !in_media && p_type == 0 &&
+                              !p_searchMETAURL &&
+                              singlefile_may_inline(
+                                  intag_start_valid ? intag_start : NULL,
+                                  tag_attr_start)) {
+                            strcatbuff(tempo, SINGLEFILE_MARK);
+                          }
+
                           // écrire le lien modifié, relatif
                           // Note: escape all chars, even >127 (no UTF)
                           HT_ADD_HTMLESCAPED_FULL(tempo);

src/htsrobots.c

@@ -147,8 +147,7 @@ static void robots_blob_add(char *blob, size_t blobsize, char marker,
 
 void robots_parse(robots_wizard *robots, const char *adr, const char *body,
                   size_t bodysize, char *info, size_t infosize,
-                  hts_boolean keep_root_disallow, char *sitemaps,
-                  size_t sitemapsize) {
+                  hts_boolean keep_root_disallow) {
   size_t bptr = 0;
   int record = 0;
   char BIGSTK line[1024];
@@ -157,8 +156,6 @@ void robots_parse(robots_wizard *robots, const char *adr, const char *body,
   blob[0] = '\0';
   if (info != NULL && infosize > 0)
     info[0] = '\0';
-  if (sitemaps != NULL && sitemapsize > 0)
-    sitemaps[0] = '\0';
 #if DEBUG_ROBOTS
   printf("robots.txt dump:\n%s\n", body);
 #endif
@@ -175,19 +172,7 @@ void robots_parse(robots_wizard *robots, const char *adr, const char *body,
       line[llen - 1] = '\0';
       llen--;
     }
-    if (sitemaps != NULL && strfield(line, "sitemap:")) {
-      // group-independent record (RFC 9309): collected whatever the group
-      char *a = line + 8;
-
-      while (is_realspace(*a))
-        a++;
-      /* A line at the buffer limit was truncated: a half URL is not one. */
-      if (strnotempty(a) && strlen(line) < sizeof(line) - 3 &&
-          strlen(a) + 2 < sitemapsize - strlen(sitemaps)) {
-        strlcatbuff(sitemaps, a, sitemapsize);
-        strlcatbuff(sitemaps, "\n", sitemapsize);
-      }
-    } else if (strfield(line, "user-agent:")) {
+    if (strfield(line, "user-agent:")) {
       char *a = line + 11;
 
       while (is_realspace(*a))

src/htsrobots.h

@@ -56,12 +56,10 @@ int checkrobots(robots_wizard * robots, const char *adr, const char *fil);
 void checkrobots_free(robots_wizard * robots);
 int checkrobots_set(robots_wizard * robots, const char *adr, const char *data);
 /* Parse robots.txt `body` for `adr`, storing the HTTrack group's rules; `info`
-   gets a disallow summary, `keep_root_disallow` FALSE drops "Disallow: /", and
-   `sitemaps` (optional) collects the Sitemap: URLs, one per line. */
+   gets a disallow summary, `keep_root_disallow` FALSE drops "Disallow: /". */
 void robots_parse(robots_wizard *robots, const char *adr, const char *body,
                   size_t bodysize, char *info, size_t infosize,
-                  hts_boolean keep_root_disallow, char *sitemaps,
-                  size_t sitemapsize);
+                  hts_boolean keep_root_disallow);
 #endif
 
 #endif

src/htssafe.h

@@ -539,68 +539,6 @@ static HTS_INLINE HTS_UNUSED HTS_PRINTF_FUN(3, 4) void slprintfbuff_clip(
 #define sprintfbuff(ARR, ...) slprintfbuff((ARR), sizeof(ARR), __VA_ARGS__)
 #endif
 
-/* Last character of s, or '\0' when s is empty. Replaces s[strlen(s) - 1],
-   which indexes one byte before the buffer on an empty string. */
-static HTS_INLINE HTS_UNUSED char hts_lastchar(const char *s) {
-  const size_t len = strlen(s);
-
-  return len != 0 ? s[len - 1] : '\0';
-}
-
-/* Drop a trailing c from s if present; HTS_TRUE if one was dropped. */
-static HTS_INLINE HTS_UNUSED hts_boolean hts_striplastchar(char *s, char c) {
-  const size_t len = strlen(s);
-
-  if (len != 0 && s[len - 1] == c) {
-    s[len - 1] = '\0';
-    return HTS_TRUE;
-  }
-  return HTS_FALSE;
-}
-
-/* Drop the last character of s whatever it is; HTS_TRUE if s was not empty. */
-static HTS_INLINE HTS_UNUSED hts_boolean hts_choplastchar(char *s) {
-  const size_t len = strlen(s);
-
-  if (len != 0) {
-    s[len - 1] = '\0';
-    return HTS_TRUE;
-  }
-  return HTS_FALSE;
-}
-
-/* htslib.h's is_space() and is_realspace() as hts_rtrim() sets; -#test=rtrim
-   keeps them in sync. */
-#define HTS_SPACES " \"\n\r\t\f\v'"
-#define HTS_REALSPACES " \n\r\t\f\v"
-
-/* Length of s once its trailing bytes from set are dropped; 0 if they all are.
-   Counts down from the end, so it stops at s rather than below the buffer. */
-static HTS_INLINE HTS_UNUSED size_t hts_rtrimlen(const char *s,
-                                                 const char *set) {
-  size_t len = strlen(s);
-
-  while (len != 0 && strchr(set, s[len - 1]) != NULL)
-    len--;
-  return len;
-}
-
-/* Drop the trailing bytes of s that occur in set. */
-static HTS_INLINE HTS_UNUSED void hts_rtrim(char *s, const char *set) {
-  s[hts_rtrimlen(s, set)] = '\0';
-}
-
-/* Offset of the last character of s, or 0 when s is empty. */
-static HTS_INLINE HTS_UNUSED size_t hts_lastcharoffset(const char *s) {
-  const size_t len = strlen(s);
-
-  return len != 0 ? len - 1 : 0;
-}
-
-/* Address of the last character of S, or of its terminating NUL when S is
-   empty. S is evaluated twice, so pass an lvalue. */
-#define hts_lastcharptr(S) ((S) + hts_lastcharoffset(S))
-
 /* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
    added bounds checking. freet() also NULLs the freed pointer and tolerates
    NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the

src/htsserver.c

@@ -1050,9 +1050,9 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                 if (!linput(fp, line, sizeof(line) - 2)) {
                   *str = '\0';
                 }
-                if (hts_lastchar(str) == '\\') {
+                if (*str && str[strlen(str) - 1] == '\\') {
                   nocr = 1;
-                  hts_striplastchar(str, '\\');
+                  str[strlen(str) - 1] = '\0';
                 }
                 while(*str) {
                   char *pos;
@@ -1169,8 +1169,11 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                           char *rpath = (char *) adr;
 
                           //find_handle h;
-                          /* note: patching stored (inhash) value */
-                          hts_striplastchar(rpath, '/');
+                          if (rpath[0]) {
+                            if (rpath[strlen(rpath) - 1] == '/') {
+                              rpath[strlen(rpath) - 1] = '\0';  /* note: patching stored (inhash) value */
+                            }
+                          }
                           {
                             const char *profiles = hts_getcategories(rpath, 0);
                             const char *categ = hts_getcategories(rpath, 1);

src/htssinglefile.c

@@ -27,9 +27,11 @@ Please visit our Website: http://www.httrack.com
 
 /* ------------------------------------------------------------ */
 /* --single-file asset inliner. See htssinglefile.h.
-   Runs on the finished tree, so resolving a reference is path arithmetic
-   clamped to the mirror root; anything else (absolute, scheme-bearing, already
-   data:) is left alone, which is what makes a second --update run a no-op. */
+   htsparse marked every inlinable reference while saving, so this is a
+   substitution over those marks: no HTML and no CSS is parsed here. Resolving
+   a mark is path arithmetic clamped to the mirror root; anything the mark
+   cannot be resolved to keeps its link, which is what makes a second --update
+   run a no-op. */
 /* ------------------------------------------------------------ */
 
 #define HTS_INTERNAL_BYTECODE
@@ -50,14 +52,9 @@ Please visit our Website: http://www.httrack.com
 #define SF_C_FONT 2
 #define SF_C_CSS 4
 #define SF_C_JS 8
-#define SF_C_ANY (SF_C_IMAGE | SF_C_FONT | SF_C_CSS | SF_C_JS)
 
-/* Shape of an attribute value. */
-#define SF_V_URL 0    /* one URL */
-#define SF_V_SRCSET 1 /* HTML srcset candidate list */
-#define SF_V_CSS 2    /* CSS declarations (style="...") */
-
-/* @import chains deeper than this keep their links. */
+/* An asset that carries marks of its own (a stylesheet importing another) is
+   expanded before encoding; deeper than this it keeps its links. */
 #define SF_MAX_CSS_DEPTH 4
 
 /* Bounds on hostile input: a longer reference, or one resolving to more
@@ -65,9 +62,6 @@ Please visit our Website: http://www.httrack.com
 #define SF_MAX_REF 4096
 #define SF_MAX_COMPONENTS 128
 
-/* Attributes collected from one start tag before it is re-emitted. */
-#define SF_MAX_ATTRS 64
-
 /* Over-cap assets reported per pass; beyond that the log would carry one line
    per referencing page. */
 #define SF_MAX_WARN 32
@@ -83,7 +77,41 @@ typedef struct sf_ctx {
 } sf_ctx;
 
 /* ------------------------------------------------------------ */
-/* Spans                                                         */
+/* Marks                                                         */
+/* ------------------------------------------------------------ */
+
+/* (tag, attribute) pairs htsparse detects that name a page or a media stream
+   rather than an asset; a NULL tag matches any. Everything else it detects is
+   offered to the pass, which decides from the referenced file's own MIME
+   type, so a new row in hts_detect[] is covered without a change here. */
+static const struct sf_deny_rule {
+  const char *tag;
+  const char *attr;
+} sf_deny_rules[] = {
+    {"a", "href"},      {"area", "href"},   {"iframe", "src"},
+    {"frame", "src"},   {"applet", "code"}, {NULL, "longdesc"},
+    {NULL, "usemap"},   {NULL, "archive"},  {NULL, "profile"},
+    {NULL, "codebase"},
+};
+
+hts_boolean singlefile_may_inline(const char *tag_start, const char *attr) {
+  size_t i;
+
+  if (attr == NULL)
+    return HTS_FALSE;
+  for (i = 0; i < sizeof(sf_deny_rules) / sizeof(sf_deny_rules[0]); i++) {
+    const struct sf_deny_rule *const r = &sf_deny_rules[i];
+
+    if (r->tag != NULL && (tag_start == NULL || !check_tag(tag_start, r->tag)))
+      continue;
+    if (rech_tageq(attr, r->attr))
+      return HTS_FALSE;
+  }
+  return HTS_TRUE;
+}
+
+/* ------------------------------------------------------------ */
+/* Paths                                                         */
 /* ------------------------------------------------------------ */
 
 static int sf_is_space(int c) {
@@ -92,69 +120,6 @@ static int sf_is_space(int c) {
 
 static int sf_is_sep(int c) { return c == '/' || c == '\\'; }
 
-/* Drop the whitespace surrounding a reference. */
-static void sf_trim(const char **ref, size_t *reflen) {
-  while (*reflen > 0 && sf_is_space((unsigned char) **ref)) {
-    (*ref)++;
-    (*reflen)--;
-  }
-  while (*reflen > 0 && sf_is_space((unsigned char) (*ref)[*reflen - 1]))
-    (*reflen)--;
-}
-
-/* Text after a reference's first '#', or NULL when it carries no fragment. */
-static const char *sf_fragment(const char *ref, size_t reflen, size_t *len) {
-  size_t i;
-
-  sf_trim(&ref, &reflen);
-  for (i = 0; i < reflen; i++) {
-    if (ref[i] == '#') {
-      *len = reflen - i - 1;
-      return ref + i + 1;
-    }
-  }
-  return NULL;
-}
-
-/* Case-insensitive equality between the span [p,p+n) and a lowercase literal.
- */
-static hts_boolean sf_span_eq(const char *p, size_t n, const char *lit) {
-  size_t i;
-
-  for (i = 0; i < n; i++) {
-    if (lit[i] == '\0' || tolower((unsigned char) p[i]) != lit[i])
-      return HTS_FALSE;
-  }
-  return lit[n] == '\0' ? HTS_TRUE : HTS_FALSE;
-}
-
-/* Case-insensitive "does the span start with the lowercase literal". */
-static hts_boolean sf_span_starts(const char *p, size_t n, const char *lit) {
-  size_t i;
-
-  for (i = 0; lit[i] != '\0'; i++) {
-    if (i >= n || tolower((unsigned char) p[i]) != lit[i])
-      return HTS_FALSE;
-  }
-  return HTS_TRUE;
-}
-
-/* Case-insensitive search for a lowercase literal inside a span. */
-static hts_boolean sf_span_has(const char *p, size_t n, const char *lit) {
-  const size_t l = strlen(lit);
-  size_t i;
-
-  for (i = 0; l <= n && i + l <= n; i++) {
-    if (sf_span_starts(p + i, n - i, lit))
-      return HTS_TRUE;
-  }
-  return HTS_FALSE;
-}
-
-/* ------------------------------------------------------------ */
-/* Paths                                                         */
-/* ------------------------------------------------------------ */
-
 /* Copy path into out with '/' separators and no trailing one. */
 static void sf_normalize_path(const char *path, String *out) {
   size_t i;
@@ -226,7 +191,12 @@ static hts_boolean sf_resolve(const sf_ctx *ctx, const char *base_dir,
   size_t i, n;
   int sp = 0, part;
 
-  sf_trim(&ref, &reflen);
+  while (reflen > 0 && sf_is_space((unsigned char) *ref)) {
+    ref++;
+    reflen--;
+  }
+  while (reflen > 0 && sf_is_space((unsigned char) ref[reflen - 1]))
+    reflen--;
   /* A mirrored name carries no query or fragment. */
   for (n = 0; n < reflen; n++) {
     if (ref[n] == '#' || ref[n] == '?')
@@ -366,20 +336,17 @@ static hts_boolean sf_append_base64(String *out, char *data, size_t len) {
   return HTS_TRUE;
 }
 
-/* Append [p,p+len), escaping what an unquoted CSS url(), an HTML attribute or
-   an appended fragment could choke on. preencoded text already carries the
-   document's own '%' and '&' escapes, and encoding those again changes it. */
-static void sf_append_escaped(String *out, const char *p, size_t len,
-                              hts_boolean preencoded) {
+/* Append a mirror-relative path, percent-escaping everything an unquoted CSS
+   url() or an HTML attribute could choke on. */
+static void sf_append_escaped_path(String *out, const char *p) {
   static const char hex[] = "0123456789ABCDEF";
   size_t i;
 
-  for (i = 0; i < len; i++) {
+  for (i = 0; p[i] != '\0'; i++) {
     const unsigned char c = (unsigned char) p[i];
 
     if (c <= 32 || c >= 127 || c == '"' || c == '\'' || c == '(' || c == ')' ||
-        c == '\\' || c == '<' || c == '>' || c == '#' ||
-        (!preencoded && (c == '%' || c == '&'))) {
+        c == '\\' || c == '<' || c == '>' || c == '&' || c == '%') {
       StringAddchar(*out, '%');
       StringAddchar(*out, hex[c >> 4]);
       StringAddchar(*out, hex[c & 15]);
@@ -389,13 +356,6 @@ static void sf_append_escaped(String *out, const char *p, size_t len,
   }
 }
 
-/* Re-attach a fragment: it selects inside the asset (an SVG sprite id) rather
-   than naming it, so the replacement needs it too. */
-static void sf_append_fragment(String *out, const char *frag, size_t len) {
-  StringAddchar(*out, '#');
-  sf_append_escaped(out, frag, len, HTS_TRUE);
-}
-
 static void sf_warn_oversize(sf_ctx *ctx, const char *path, LLint size,
                              LLint cap) {
   if (*ctx->warn_budget <= 0)
@@ -411,25 +371,25 @@ static void sf_warn_oversize(sf_ctx *ctx, const char *path, LLint size,
                 path, size, cap);
 }
 
-static void sf_rewrite_css(sf_ctx *ctx, const char *base_dir, int depth,
-                           const char *css, size_t len, String *out);
+/* ------------------------------------------------------------ */
+/* Substitution                                                  */
+/* ------------------------------------------------------------ */
+
+static void sf_expand(sf_ctx *ctx, const char *base_dir, int depth,
+                      const char *body, size_t len, String *out);
 
 /* Replace the reference [ref,ref+reflen), resolved against base_dir, with its
-   data: URI appended to out. classes gates the acceptable MIME classes;
-   fallback_mime types an asset whose class cannot be guessed (NULL: give up).
-   rebase_dir re-expresses an un-inlinable asset relative to that directory.
-   A data: URL's path is opaque, so nothing relative inside an inlined
-   stylesheet resolves anyway; this only aims it at a lenient resolver's base.
-   Returns HTS_TRUE if out received a replacement. */
+   data: URI appended to out. rebase_dir re-expresses an un-inlinable asset
+   relative to that directory: a data: URL's path is opaque, so nothing
+   relative inside an inlined stylesheet resolves anyway; this only aims it at
+   a lenient resolver's base. Returns HTS_TRUE if out received a data: URI. */
 static hts_boolean sf_inline(sf_ctx *ctx, const char *base_dir, const char *ref,
-                             size_t reflen, int classes,
-                             const char *fallback_mime, const char *rebase_dir,
-                             int depth, String *out) {
+                             size_t reflen, const char *rebase_dir, int depth,
+                             String *out) {
   String path = STRING_EMPTY;
   char mime[HTS_MIMETYPE_SIZE];
-  char *body;
-  size_t body_len = 0, frag_len = 0;
-  const char *const frag = sf_fragment(ref, reflen, &frag_len);
+  char *file;
+  size_t file_len = 0;
   LLint size, cap;
   int cls;
   hts_boolean done = HTS_FALSE;
@@ -439,13 +399,9 @@ static hts_boolean sf_inline(sf_ctx *ctx, const char *base_dir, const char *ref,
     return HTS_FALSE;
   }
   cls = sf_mime_class(ctx->opt, StringBuff(path), mime, sizeof(mime));
-  if (cls == 0 && fallback_mime != NULL) {
-    cls = classes;
-    strlcpybuff(mime, fallback_mime, sizeof(mime));
-  }
   cap = ctx->opt->single_file_max_size;
   size = fsize_utf8(StringBuff(path));
-  if ((cls & classes) == 0 || size < 0)
+  if (cls == 0 || size < 0)
     goto fallback;
   if (size > cap || size > SINGLEFILE_HARD_MAX_SIZE) {
     sf_warn_oversize(ctx, StringBuff(path), size, cap);
@@ -455,11 +411,11 @@ static hts_boolean sf_inline(sf_ctx *ctx, const char *base_dir, const char *ref,
     sf_warn_oversize(ctx, StringBuff(path), size, ctx->budget);
     goto fallback;
   }
-  body = sf_readfile(StringBuff(path), &body_len);
-  if (body == NULL)
+  file = sf_readfile(StringBuff(path), &file_len);
+  if (file == NULL)
     goto fallback;
-  /* Charged before the nested rewrite: an @import chain otherwise spends what
-     its ancestors already claimed, and the budget ends up negative. */
+  /* Charged before the nested expansion: an @import chain otherwise spends
+     what its ancestors already claimed, and the budget ends up negative. */
   ctx->budget -= size;
   /* Encode into a scratch String: a failed encode must leave out untouched,
      not a truncated "data:...;base64," with no payload. */
@@ -467,32 +423,30 @@ static hts_boolean sf_inline(sf_ctx *ctx, const char *base_dir, const char *ref,
     String payload = STRING_EMPTY;
 
     StringClear(payload);
-    if ((cls & SF_C_CSS) != 0 && depth < SF_MAX_CSS_DEPTH) {
+    if ((cls & (SF_C_CSS | SF_C_JS)) != 0 && depth < SF_MAX_CSS_DEPTH) {
       String nested = STRING_EMPTY;
       String nested_dir = STRING_EMPTY;
 
       sf_dirname(StringBuff(path), &nested_dir);
       StringClear(nested);
-      sf_rewrite_css(ctx, StringBuff(nested_dir), depth + 1, body, body_len,
-                     &nested);
+      sf_expand(ctx, StringBuff(nested_dir), depth + 1, file, file_len,
+                &nested);
       done = sf_append_base64(&payload, StringBuffRW(nested),
                               StringLength(nested));
       StringFree(nested);
       StringFree(nested_dir);
     } else {
-      done = sf_append_base64(&payload, body, body_len);
+      done = sf_append_base64(&payload, file, file_len);
     }
     if (done) {
       StringCat(*out, "data:");
       StringCat(*out, mime);
       StringCat(*out, ";base64,");
       StringMemcat(*out, StringBuff(payload), StringLength(payload));
-      if (frag != NULL)
-        sf_append_fragment(out, frag, frag_len);
     }
     StringFree(payload);
   }
-  freet(body);
+  freet(file);
   if (done) {
     ctx->inlined++;
     StringFree(path);
@@ -507,9 +461,7 @@ fallback:
     StringClear(rel);
     sf_relative_from(rebase_dir, StringBuff(path), &rel);
     if (StringLength(rel) > 0) {
-      sf_append_escaped(out, StringBuff(rel), StringLength(rel), HTS_FALSE);
-      if (frag != NULL)
-        sf_append_fragment(out, frag, frag_len);
+      sf_append_escaped_path(out, StringBuff(rel));
       done = HTS_TRUE;
     }
     StringFree(rel);
@@ -518,547 +470,43 @@ fallback:
   return done;
 }
 
-/* ------------------------------------------------------------ */
-/* CSS                                                           */
-/* ------------------------------------------------------------ */
-
-/* Rewrite the CSS in [css,css+len) into out, resolving its references against
-   base_dir. depth counts the @import nesting. */
-static void sf_rewrite_css(sf_ctx *ctx, const char *base_dir, int depth,
-                           const char *css, size_t len, String *out) {
-  size_t i = 0;
-  int importing = 0; /* set for exactly one iteration: see the @import branch */
-
-  while (i < len) {
-    const int is_import = importing;
-
-    importing = 0;
-    /* A comment or a string is copied verbatim: a url( inside either is not a
-       reference. */
-    if (css[i] == '/' && i + 1 < len && css[i + 1] == '*') {
-      const size_t start = i;
-
-      i += 2;
-      while (i + 1 < len && !(css[i] == '*' && css[i + 1] == '/'))
-        i++;
-      i = i + 1 < len ? i + 2 : len;
-      StringMemcat(*out, css + start, i - start);
-      continue;
-    }
-    if (css[i] == '@' && sf_span_starts(css + i, len - i, "@import")) {
-      size_t j = i + 7;
-
-      while (j < len && sf_is_space((unsigned char) css[j]))
-        j++;
-      /* url(...) here names a stylesheet, so hand it to the url( branch with
-         the import's class rather than the image/font one. */
-      if (j < len && sf_span_starts(css + j, len - j, "url(")) {
-        StringMemcat(*out, css + i, j - i);
-        i = j;
-        importing = 1;
-        continue;
-      }
-      if (j < len && (css[j] == '"' || css[j] == '\'')) {
-        const char quote = css[j];
-        const size_t vstart = j + 1;
-        size_t vend = vstart;
-
-        while (vend < len && css[vend] != quote && css[vend] != '\n') {
-          if (css[vend] == '\\' && vend + 1 < len)
-            vend++; /* an escaped quote does not end the string */
-          vend++;
-        }
-        if (vend < len && css[vend] == quote) {
-          String repl = STRING_EMPTY;
-
-          StringClear(repl);
-          StringMemcat(*out, css + i, j - i);
-          if (sf_inline(ctx, base_dir, css + vstart, vend - vstart, SF_C_CSS,
-                        "text/css", NULL, depth, &repl)) {
-            StringCat(*out, "\"");
-            StringMemcat(*out, StringBuff(repl), StringLength(repl));
-            StringCat(*out, "\"");
-          } else {
-            StringMemcat(*out, css + j, vend + 1 - j);
-          }
-          StringFree(repl);
-          i = vend + 1;
-          continue;
-        }
-      }
-      StringMemcat(*out, css + i, 7);
-      i += 7;
-      continue;
-    }
-    if ((css[i] == 'u' || css[i] == 'U') &&
-        sf_span_starts(css + i, len - i, "url(") &&
-        (i == 0 || (!isalnum((unsigned char) css[i - 1]) && css[i - 1] != '-' &&
-                    css[i - 1] != '_'))) {
-      const size_t start = i;
-      size_t j = i + 4, vstart, vend;
-      char quote = '\0';
-
-      while (j < len && sf_is_space((unsigned char) css[j]))
-        j++;
-      if (j < len && (css[j] == '"' || css[j] == '\'')) {
-        quote = css[j];
-        j++;
-      }
-      vstart = j;
-      while (j < len && (quote != '\0' ? css[j] != quote : css[j] != ')') &&
-             css[j] != '\n') {
-        if (quote != '\0' && css[j] == '\\' && j + 1 < len)
-          j++; /* an escaped quote does not end the string */
-        j++;
-      }
-      vend = j;
-      if (quote != '\0' && j < len && css[j] == quote) {
-        j++;
-        while (j < len && sf_is_space((unsigned char) css[j]))
-          j++;
-      }
-      if (j < len && css[j] == ')') {
-        String repl = STRING_EMPTY;
-
-        j++;
-        StringClear(repl);
-        /* Emit unquoted: the CSS could be a style="..." attribute value, where
-           a quote would end the attribute. A data: payload and an escaped path
-           both stay inside the unquoted url-token alphabet. */
-        if (sf_inline(ctx, base_dir, css + vstart, vend - vstart,
-                      is_import ? SF_C_CSS : SF_C_IMAGE | SF_C_FONT,
-                      is_import ? "text/css" : NULL,
-                      !is_import && depth > 0 ? ctx->page_dir : NULL, depth,
-                      &repl)) {
-          StringCat(*out, "url(");
-          StringMemcat(*out, StringBuff(repl), StringLength(repl));
-          StringCat(*out, ")");
-        } else {
-          StringMemcat(*out, css + start, j - start);
-        }
-        StringFree(repl);
-        i = j;
-        continue;
-      }
-      StringMemcat(*out, css + start, 4); /* unterminated url(: leave it */
-      i = start + 4;
-      continue;
-    }
-    if (css[i] == '"' || css[i] == '\'') {
-      const char quote = css[i];
-      const size_t start = i;
-
-      i++;
-      while (i < len && css[i] != quote && css[i] != '\n') {
-        if (css[i] == '\\' && i + 1 < len)
-          i++;
-        i++;
-      }
-      if (i < len && css[i] == quote)
-        i++;
-      StringMemcat(*out, css + start, i - start);
-      continue;
-    }
-    StringAddchar(*out, css[i]);
-    i++;
-  }
+/* The mark ends a reference; these end the token the mark was appended to. */
+static int sf_is_ref_delim(int c) {
+  return sf_is_space(c) || c == '"' || c == '\'' || c == '(' || c == ')' ||
+         c == '=' || c == ',' || c == '<' || c == '>' || c == ';';
 }
 
-/* ------------------------------------------------------------ */
-/* HTML                                                          */
-/* ------------------------------------------------------------ */
+/* Copy [body,body+len) to out, replacing each marked reference by its data:
+   URI, or by the bare reference when it cannot be inlined. */
+static void sf_expand(sf_ctx *ctx, const char *base_dir, int depth,
+                      const char *body, size_t len, String *out) {
+  const size_t marklen = strlen(SINGLEFILE_MARK);
+  size_t i = 0, flushed = 0;
 
-/* srcset is "url [descriptor]" candidates separated by commas. Per the HTML
-   candidate-parsing rules the URL token runs to the next whitespace and may
-   carry trailing commas, so an emitted "data:...;base64,AAAA 2x" round-trips.
- */
-static void sf_rewrite_srcset(sf_ctx *ctx, const char *v, size_t len,
-                              String *out) {
-  size_t i = 0;
-  int first = 1;
+  while (i + marklen <= len) {
+    size_t start, tail;
 
-  while (i < len) {
-    size_t ustart, uend, dstart, dend;
-
-    while (i < len && (sf_is_space((unsigned char) v[i]) || v[i] == ','))
-      i++;
-    if (i >= len)
-      break;
-    ustart = i;
-    while (i < len && !sf_is_space((unsigned char) v[i]))
-      i++;
-    uend = i;
-    while (uend > ustart && v[uend - 1] == ',')
-      uend--;
-    dstart = dend = i;
-    if (uend == i) { /* no trailing comma, so a descriptor may follow */
-      while (i < len && v[i] != ',')
-        i++;
-      dstart = uend;
-      dend = i;
-      while (dend > dstart && sf_is_space((unsigned char) v[dend - 1]))
-        dend--;
-      while (dstart < dend && sf_is_space((unsigned char) v[dstart]))
-        dstart++;
-    }
-    if (!first)
-      StringCat(*out, ", ");
-    first = 0;
-    if (!sf_inline(ctx, ctx->page_dir, v + ustart, uend - ustart, SF_C_IMAGE,
-                   NULL, NULL, 0, out))
-      StringMemcat(*out, v + ustart, uend - ustart);
-    if (dend > dstart) {
-      StringAddchar(*out, ' ');
-      StringMemcat(*out, v + dstart, dend - dstart);
-    }
-  }
-}
-
-/* (tag, attribute) pairs naming an inlinable asset; a NULL tag matches any.
-   Deliberately absent though hts_detect[] downloads them: a@href, iframe@src
-   and longdesc name pages, dynsrc and non-image source@src name media,
-   track@src has no inlinable MIME class, usemap is a same-document fragment,
-   archive lists applet jars. link@href is decided from its rel
-   (sf_link_classes); base@href needs no rule, htsparse drops it when saving. */
-static const struct sf_attr_rule {
-  const char *tag;
-  const char *attr;
-  int classes;
-  int shape;
-} sf_attr_rules[] = {
-    {"img", "src", SF_C_IMAGE, SF_V_URL},
-    {"img", "srcset", SF_C_IMAGE, SF_V_SRCSET},
-    {"source", "src", SF_C_IMAGE, SF_V_URL},
-    {"source", "srcset", SF_C_IMAGE, SF_V_SRCSET},
-    {"input", "src", SF_C_IMAGE, SF_V_URL},
-    {"video", "poster", SF_C_IMAGE, SF_V_URL},
-    {"image", "href", SF_C_IMAGE, SF_V_URL},
-    {"image", "xlink:href", SF_C_IMAGE, SF_V_URL},
-    {"object", "data", SF_C_IMAGE, SF_V_URL},
-    {"embed", "src", SF_C_IMAGE, SF_V_URL},
-    {"script", "src", SF_C_JS, SF_V_URL},
-    /* Lazy loading: src is a placeholder and the real image rides one of
-       these. The image class keeps a lazy <iframe> or <script> a link. */
-    {NULL, "data-src", SF_C_IMAGE, SF_V_URL},
-    {NULL, "data-srcset", SF_C_IMAGE, SF_V_SRCSET},
-    {NULL, "lowsrc", SF_C_IMAGE, SF_V_URL},
-    {NULL, "background", SF_C_IMAGE, SF_V_URL},
-    {NULL, "style", 0, SF_V_CSS},
-};
-
-typedef struct sf_attr {
-  const char *pre; /* whitespace before the attribute */
-  size_t pre_len;
-  const char *raw; /* name plus value, with the original quoting */
-  size_t raw_len;
-  const char *name;
-  size_t name_len;
-  const char *value;
-  size_t value_len;
-  hts_boolean has_value;
-} sf_attr;
-
-/* Accepted classes and fallback type for <link href>, read from its rel: a
-   navigational rel (canonical, alternate, next) yields 0 and keeps the link. */
-static int sf_link_classes(const sf_attr *attrs, int nattrs,
-                           const char **fallback) {
-  int i;
-
-  *fallback = NULL;
-  for (i = 0; i < nattrs; i++) {
-    if (!sf_span_eq(attrs[i].name, attrs[i].name_len, "rel"))
-      continue;
-    if (sf_span_has(attrs[i].value, attrs[i].value_len, "stylesheet")) {
-      *fallback = "text/css";
-      return SF_C_CSS;
-    }
-    if (sf_span_has(attrs[i].value, attrs[i].value_len, "icon"))
-      return SF_C_IMAGE;
-    if (sf_span_has(attrs[i].value, attrs[i].value_len, "preload") ||
-        sf_span_has(attrs[i].value, attrs[i].value_len, "prefetch"))
-      return SF_C_ANY;
-    return 0;
-  }
-  return 0;
-}
-
-static const struct sf_attr_rule *sf_find_rule(const char *tag, size_t tag_len,
-                                               const char *attr,
-                                               size_t attr_len) {
-  size_t i;
-
-  for (i = 0; i < sizeof(sf_attr_rules) / sizeof(sf_attr_rules[0]); i++) {
-    const struct sf_attr_rule *const r = &sf_attr_rules[i];
-
-    if (r->tag != NULL && !sf_span_eq(tag, tag_len, r->tag))
-      continue;
-    if (sf_span_eq(attr, attr_len, r->attr))
-      return r;
-  }
-  return NULL;
-}
-
-/* Emit one attribute, substituting its value when a rule applies. */
-static void sf_emit_attr(sf_ctx *ctx, const char *tag, size_t tag_len,
-                         const sf_attr *a, int link_classes,
-                         const char *link_fallback, String *out) {
-  const struct sf_attr_rule *rule = NULL;
-  const char *fallback = NULL;
-  String repl = STRING_EMPTY;
-  hts_boolean done = HTS_FALSE;
-  int classes;
-
-  StringMemcat(*out, a->pre, a->pre_len);
-  if (!a->has_value) {
-    StringMemcat(*out, a->raw, a->raw_len);
-    return;
-  }
-  if (sf_span_eq(tag, tag_len, "link") &&
-      sf_span_eq(a->name, a->name_len, "href")) {
-    classes = link_classes;
-    fallback = link_fallback;
-  } else {
-    rule = sf_find_rule(tag, tag_len, a->name, a->name_len);
-    classes = rule != NULL ? rule->classes : 0;
-    if (rule != NULL && rule->classes == SF_C_JS)
-      fallback = "text/javascript";
-  }
-  if (rule == NULL && classes == 0) {
-    StringMemcat(*out, a->raw, a->raw_len);
-    return;
-  }
-  StringClear(repl);
-  if (rule != NULL && (rule->shape == SF_V_SRCSET || rule->shape == SF_V_CSS)) {
-    /* These always produce a value; substitute only when it really differs, so
-       an untouched attribute keeps its original spelling and quoting. */
-    if (rule->shape == SF_V_SRCSET)
-      sf_rewrite_srcset(ctx, a->value, a->value_len, &repl);
-    else
-      sf_rewrite_css(ctx, ctx->page_dir, 0, a->value, a->value_len, &repl);
-    done = StringLength(repl) != a->value_len ||
-                   memcmp(StringBuff(repl), a->value, a->value_len) != 0
-               ? HTS_TRUE
-               : HTS_FALSE;
-  } else {
-    done = sf_inline(ctx, ctx->page_dir, a->value, a->value_len, classes,
-                     fallback, NULL, 0, &repl);
-  }
-  if (done) {
-    size_t n;
-
-    /* Always re-quote with '"', so a value carrying a literal quote (a CSS
-       declaration that was single-quoted in the source) must be escaped. */
-    StringMemcat(*out, a->name, a->name_len);
-    StringCat(*out, "=\"");
-    for (n = 0; n < StringLength(repl); n++) {
-      const char c = StringSub(repl, n);
-
-      if (c == '"')
-        StringCat(*out, "&quot;");
-      else
-        StringAddchar(*out, c);
-    }
-    StringCat(*out, "\"");
-  } else {
-    StringMemcat(*out, a->raw, a->raw_len);
-  }
-  StringFree(repl);
-}
-
-/* Elements whose content is raw text, never markup. */
-static const char *const sf_rawtext_tags[] = {"script", "style", "textarea",
-                                              "title"};
-
-/* The raw-text tag matching [tag,tag+len), or NULL. */
-static const char *sf_rawtext_tag(const char *tag, size_t len) {
-  size_t i;
-
-  for (i = 0; i < sizeof(sf_rawtext_tags) / sizeof(sf_rawtext_tags[0]); i++) {
-    if (sf_span_eq(tag, len, sf_rawtext_tags[i]))
-      return sf_rawtext_tags[i];
-  }
-  return NULL;
-}
-
-static void sf_rewrite_html_(sf_ctx *ctx, const char *p, size_t len,
-                             String *out) {
-  size_t i = 0;
-
-  while (i < len) {
-    sf_attr attrs[SF_MAX_ATTRS];
-    const char *tag, *rawtext, *link_fallback = NULL;
-    size_t tag_len, tag_start;
-    int nattrs = 0, k, link_classes = 0;
-    hts_boolean selfclose = HTS_FALSE, overflow = HTS_FALSE;
-
-    if (p[i] != '<') {
-      StringAddchar(*out, p[i]);
+    if (memcmp(body + i, SINGLEFILE_MARK, marklen) != 0) {
       i++;
       continue;
     }
-    if (i + 4 <= len && memcmp(p + i, "<!--", 4) == 0) {
-      const size_t start = i;
-
-      i += 4;
-      /* "<!-->" and "<!--->" are empty comments, not unterminated ones. */
-      if (i < len && p[i] == '>')
-        i++;
-      else if (i + 2 <= len && memcmp(p + i, "->", 2) == 0)
-        i += 2;
-      else {
-        while (i + 3 <= len && memcmp(p + i, "-->", 3) != 0)
-          i++;
-        i = i + 3 <= len ? i + 3 : len;
-      }
-      StringMemcat(*out, p + start, i - start);
-      continue;
-    }
-    if (i + 1 >= len || !isalpha((unsigned char) p[i + 1])) {
-      const size_t start = i; /* doctype, processing instruction, end tag */
-
-      i++;
-      while (i < len && p[i] != '>') {
-        /* A '>' inside a quoted value does not close the tag. */
-        if (p[i] == '"' || p[i] == '\'') {
-          const char q = p[i++];
-
-          while (i < len && p[i] != q)
-            i++;
-        }
-        if (i < len)
-          i++;
-      }
-      if (i < len)
-        i++;
-      StringMemcat(*out, p + start, i - start);
-      continue;
-    }
-
-    /* Start tag. Collect every attribute before deciding: <link href> depends
-       on the rel, which may come after it. */
-    tag_start = i;
-    i++;
-    tag = p + i;
-    while (i < len && !sf_is_space((unsigned char) p[i]) && p[i] != '>' &&
-           p[i] != '/')
-      i++;
-    tag_len = (size_t) (p + i - tag);
-    while (i < len) {
-      const char *const pre = p + i;
-      size_t pre_len;
-      sf_attr *a;
-
-      while (i < len && sf_is_space((unsigned char) p[i]))
-        i++;
-      pre_len = (size_t) (p + i - pre);
-      if (i >= len || p[i] == '>' ||
-          (p[i] == '/' && i + 1 < len && p[i + 1] == '>')) {
-        i = (size_t) (pre - p); /* the caller copies this whitespace */
-        break;
-      }
-      if (nattrs == SF_MAX_ATTRS) {
-        overflow = HTS_TRUE; /* keep parsing, but stop recording */
-        nattrs--;
-      }
-      a = &attrs[nattrs++];
-      a->pre = pre;
-      a->pre_len = pre_len;
-      a->name = p + i;
-      a->raw = p + i;
-      a->has_value = HTS_FALSE;
-      a->value = NULL;
-      a->value_len = 0;
-      while (i < len && !sf_is_space((unsigned char) p[i]) && p[i] != '=' &&
-             p[i] != '>' && !(p[i] == '/' && i + 1 < len && p[i + 1] == '>'))
-        i++;
-      a->name_len = (size_t) (p + i - a->name);
-      if (a->name_len == 0) { /* a stray '=' or '/': copy it and move on */
-        i++;
-        a->name_len = 1;
-        a->raw_len = 1;
-        continue;
-      }
-      {
-        const size_t save = i;
-
-        while (i < len && sf_is_space((unsigned char) p[i]))
-          i++;
-        if (i >= len || p[i] != '=') {
-          i = save;
-          a->raw_len = a->name_len;
-          continue;
-        }
-      }
-      i++;
-      while (i < len && sf_is_space((unsigned char) p[i]))
-        i++;
-      if (i < len && (p[i] == '"' || p[i] == '\'')) {
-        const char quote = p[i];
-
-        i++;
-        a->value = p + i;
-        while (i < len && p[i] != quote)
-          i++;
-        a->value_len = (size_t) (p + i - a->value);
-        if (i < len)
-          i++;
-      } else {
-        a->value = p + i;
-        while (i < len && !sf_is_space((unsigned char) p[i]) && p[i] != '>')
-          i++;
-        a->value_len = (size_t) (p + i - a->value);
-      }
-      a->has_value = HTS_TRUE;
-      a->raw_len = (size_t) (p + i - a->raw);
-    }
-    if (overflow) { /* pathological tag: copy the whole of it untouched */
-      StringMemcat(*out, p + tag_start, i - tag_start);
-    } else {
-      if (sf_span_eq(tag, tag_len, "link"))
-        link_classes = sf_link_classes(attrs, nattrs, &link_fallback);
-      StringAddchar(*out, '<');
-      StringMemcat(*out, tag, tag_len);
-      for (k = 0; k < nattrs; k++)
-        sf_emit_attr(ctx, tag, tag_len, &attrs[k], link_classes, link_fallback,
-                     out);
-    }
-    while (i < len && sf_is_space((unsigned char) p[i])) {
-      StringAddchar(*out, p[i]);
-      i++;
-    }
-    if (i < len && p[i] == '/') {
-      selfclose = HTS_TRUE;
-      StringAddchar(*out, '/');
-      i++;
-    }
-    if (i < len && p[i] == '>') {
-      StringAddchar(*out, '>');
-      i++;
-    }
-
-    rawtext = selfclose ? NULL : sf_rawtext_tag(tag, tag_len);
-    if (rawtext != NULL) {
-      const size_t body = i;
-      size_t end = i;
-
-      while (end < len) {
-        if (p[end] == '<' && end + 2 < len && p[end + 1] == '/' &&
-            sf_span_starts(p + end + 2, len - end - 2, rawtext)) {
-          const size_t after = end + 2 + strlen(rawtext);
-
-          if (after >= len || sf_is_space((unsigned char) p[after]) ||
-              p[after] == '>' || p[after] == '/')
-            break;
-        }
-        end++;
-      }
-      if (sf_span_eq(tag, tag_len, "style"))
-        sf_rewrite_css(ctx, ctx->page_dir, 0, p + body, end - body, out);
-      else
-        StringMemcat(*out, p + body, end - body);
-      i = end;
+    for (start = i; start > flushed && !sf_is_ref_delim(body[start - 1]);
+         start--)
+      ;
+    /* htsparse writes the fragment and the kept query string after the mark;
+       a data: URI has no use for either. */
+    for (tail = i + marklen; tail < len && !sf_is_ref_delim(body[tail]); tail++)
+      ;
+    StringMemcat(*out, body + flushed, start - flushed);
+    if (!sf_inline(ctx, base_dir, body + start, i - start,
+                   depth > 0 ? ctx->page_dir : NULL, depth, out)) {
+      StringMemcat(*out, body + start, i - start);
+      StringMemcat(*out, body + i + marklen, tail - i - marklen);
     }
+    i = tail;
+    flushed = i;
   }
+  StringMemcat(*out, body + flushed, len - flushed);
 }
 
 /* ------------------------------------------------------------ */
@@ -1085,15 +533,7 @@ hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
   ctx.warn_budget = &budget;
   ctx.budget = page_budget;
   ctx.inlined = 0;
-  /* A UTF-16/32 page is not ASCII-delimited, so byte scanning would corrupt
-     it; every ASCII-compatible charset is safe. */
-  if (html_len >= 2 &&
-      (((unsigned char) html[0] == 0xff && (unsigned char) html[1] == 0xfe) ||
-       ((unsigned char) html[0] == 0xfe && (unsigned char) html[1] == 0xff))) {
-    StringMemcat(*out, html, html_len);
-  } else {
-    sf_rewrite_html_(&ctx, html, html_len, out);
-  }
+  sf_expand(&ctx, StringBuff(dir), 0, html, html_len, out);
   StringFree(nroot);
   StringFree(npage);
   StringFree(dir);
@@ -1142,8 +582,13 @@ hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
       (void) chmod(fconv(catbuff, sizeof(catbuff), StringBuff(tmp)),
                    HTS_ACCESS_FILE);
 #endif
-    if (ok)
-      ok = hts_rename_over(opt, StringBuff(tmp), page_path);
+    if (ok) {
+      /* RENAME does not clobber an existing target on Windows. */
+      if (RENAME(StringBuff(tmp), page_path) != 0) {
+        (void) UNLINK(page_path);
+        ok = RENAME(StringBuff(tmp), page_path) == 0 ? HTS_TRUE : HTS_FALSE;
+      }
+    }
     if (!ok) {
       hts_log_print(opt, LOG_ERROR, "single-file: could not rewrite %s",
                     page_path);

src/htssinglefile.h

@@ -55,18 +55,27 @@ extern "C" {
    so a few hundred bytes of hostile CSS can otherwise ask for gigabytes. */
 #define SINGLEFILE_MAX_PAGE_SIZE (64 * 1024 * 1024)
 
+/* Fragment htsparse appends to a saved reference the pass may inline. A
+   fragment and not a scheme, so the mirror still resolves if the pass never
+   runs, and so a mirrored .css or .js keeps its marks across an --update. */
+#define SINGLEFILE_MARK "#!htsinline"
+
+/* HTS_TRUE if a reference htsparse detected in this context may become a
+   data: URI. tag_start points at the '<' of the enclosing start tag, or NULL
+   when there is none (inside a stylesheet or a script); attr at the attribute
+   name. Everything htsparse detects is inlinable unless it names a page. */
+hts_boolean singlefile_may_inline(const char *tag_start, const char *attr);
+
 /* Rewrite every HTML page the mirror produced. No-op unless opt->single_file;
    call once the tree is final, after the update purge. */
 void singlefile_process_mirror(httrackp *opt);
 
-/* Rewrite one HTML document held in memory, appending the result to out.
-   root is the mirror directory that references may not escape; page_path is
-   the document's own path under it (both UTF-8, '/' or native separators).
-   page_budget caps the total inlined bytes, since nested @import fans out
+/* Expand the marks in the document held in memory, appending the result to
+   out. root is the mirror directory that references may not escape; page_path
+   is the document's own path under it (both UTF-8, '/' or native separators).
+   page_budget caps the total inlined bytes, since a nested @import fans out
    multiplicatively; the mirror pass passes SINGLEFILE_MAX_PAGE_SIZE.
-   Returns HTS_TRUE if at least one reference was replaced; out may still
-   differ from the input when that is HTS_FALSE, since a style or srcset value
-   is re-serialized in place. */
+   Returns HTS_TRUE if at least one reference was replaced. */
 hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
                                     const char *page_path, const char *html,
                                     size_t html_len, LLint page_budget,

src/htssitemap.c

@@ -1,615 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* ------------------------------------------------------------ */
-/* File: sitemap ingestion (sitemaps.org 0.9)                   */
-/* Author: Xavier Roche                                         */
-/* ------------------------------------------------------------ */
-
-#define HTS_INTERNAL_BYTECODE
-
-#include "htscore.h"
-#include "htssitemap.h"
-
-#include "htsbase.h"
-#include "htscodec.h"
-#include "htsencoding.h"
-#include "htsfilters.h"
-#include "htshash.h"
-#include "htsmodules.h"
-#include "htslib.h"
-#include "htsrobots.h"
-#include "htssafe.h"
-#include "htstools.h"
-
-#include <ctype.h>
-#include <string.h>
-
-/* One queued sitemap document awaiting ingestion. */
-typedef struct sitemap_doc {
-  char adr[HTS_URLMAXSIZE];
-  char fil[HTS_URLMAXSIZE];
-  int level;
-  hts_sitemap_source src;
-  hts_boolean done;
-  struct sitemap_doc *next;
-} sitemap_doc;
-
-struct hts_sitemap_state {
-  sitemap_doc *docs;
-  int ndocs; /* documents queued, capped by HTS_SITEMAP_MAX_DOCS */
-  int nurls; /* URLs seeded, capped by HTS_SITEMAP_MAX_URLS_TOTAL */
-  hts_boolean probe_done;    /* the robots.txt probe has been answered */
-  hts_boolean fallback_done; /* the /sitemap.xml fallback was already queued */
-  /* The crawl's own start URL. Seeded URLs are judged against it, so a site
-     cannot widen a subtree crawl by putting its sitemap at the root. */
-  char anchor_adr[HTS_URLMAXSIZE];
-  char anchor_fil[HTS_URLMAXSIZE];
-};
-typedef struct hts_sitemap_state hts_sitemap_state;
-
-/* --------------------------------------------------------------------- */
-/* Document parsing (no engine state: fuzzable and self-testable)         */
-/* --------------------------------------------------------------------- */
-
-/* Accept only an absolute http(s) URL with no space or control byte. */
-static hts_boolean sitemap_url_ok(const char *url) {
-  const char *p;
-
-  if (!strfield(url, "http://") && !strfield(url, "https://"))
-    return HTS_FALSE;
-  for (p = url; *p != '\0'; p++) {
-    if ((unsigned char) *p <= ' ' || (unsigned char) *p == 0x7f)
-      return HTS_FALSE;
-  }
-  return HTS_TRUE;
-}
-
-/* Skip to the character after the next '>' at or after p, or NULL. */
-static const char *sitemap_tag_end(const char *p, const char *end) {
-  while (p < end && *p != '>')
-    p++;
-  return p < end ? p + 1 : NULL;
-}
-
-/* HTS_TRUE when the document's root element is `name`. Skips the XML
-   declaration, comments and processing instructions first, so a comment
-   mentioning the other root element cannot decide the document type. */
-static hts_boolean sitemap_root_is(const char *doc, size_t size,
-                                   const char *name) {
-  const size_t nlen = strlen(name);
-  size_t i = 0;
-
-  if (size >= 3 && memcmp(doc, "\xef\xbb\xbf", 3) == 0)
-    i = 3; /* UTF-8 BOM */
-  while (i < size) {
-    if (isspace((unsigned char) doc[i])) {
-      i++;
-    } else if (doc[i] != '<') {
-      return HTS_FALSE; /* character data before any element: not XML */
-    } else if (i + 4 <= size && memcmp(doc + i, "<!--", 4) == 0) {
-      const char *const e = hts_memstr(doc + i, size - i, "-->", 3);
-
-      if (e == NULL)
-        return HTS_FALSE;
-      i = (size_t) (e - doc) + 3;
-    } else if (i + 2 <= size && (doc[i + 1] == '?' || doc[i + 1] == '!')) {
-      while (i < size && doc[i] != '>')
-        i++;
-      i++;
-    } else {
-      size_t j = i + 1;
-
-      /* an optional namespace prefix: <sm:sitemapindex> is the same element */
-      while (j < size && doc[j] != ':' && doc[j] != '>' &&
-             !isspace((unsigned char) doc[j]))
-        j++;
-      if (j >= size || doc[j] != ':')
-        j = i + 1;
-      else
-        j++;
-      return j + nlen <= size && memcmp(doc + j, name, nlen) == 0 &&
-                     (j + nlen == size ||
-                      isspace((unsigned char) doc[j + nlen]) ||
-                      doc[j + nlen] == '>' || doc[j + nlen] == '/')
-                 ? HTS_TRUE
-                 : HTS_FALSE;
-    }
-  }
-  return HTS_FALSE;
-}
-
-/* Decompress a gzip-framed body into a fresh buffer. The 64 MiB cap is what
-   binds in practice; deflate tops out near 1032:1, so the tree's codec budget
-   only matters as the shared policy for a coding that could go further. */
-static char *sitemap_gunzip(const char *body, size_t size, size_t *outsize) {
-  const LLint budget = hts_codec_maxout((LLint) size);
-  size_t cap = budget < (LLint) HTS_SITEMAP_MAX_BYTES
-                   ? (size_t) budget
-                   : (size_t) HTS_SITEMAP_MAX_BYTES;
-  char *out;
-  size_t n;
-
-  if (cap == 0)
-    return NULL;
-  out = malloct(cap + 1);
-  if (out == NULL)
-    return NULL;
-  n = hts_codec_head(HTS_CODEC_DEFLATE, body, size, out, cap);
-  if (n == 0) {
-    freet(out);
-    return NULL;
-  }
-  out[n] = '\0';
-  *outsize = n;
-  return out;
-}
-
-int hts_sitemap_scan(const char *body, size_t size, int maxurls,
-                     hts_boolean *is_index, hts_sitemap_handler handler,
-                     void *arg) {
-  char *unpacked = NULL;
-  const char *doc;
-  const char *end;
-  const char *p;
-  int n = 0;
-
-  if (is_index != NULL)
-    *is_index = HTS_FALSE;
-  if (body == NULL || size < 2 || handler == NULL)
-    return 0;
-
-  /* Content-Encoding gzip is undone upstream; only the container is left. */
-  if ((unsigned char) body[0] == 0x1f && (unsigned char) body[1] == 0x8b) {
-    unpacked = sitemap_gunzip(body, size, &size);
-    if (unpacked == NULL)
-      return -1;
-    doc = unpacked;
-  } else {
-    if (size > (size_t) HTS_SITEMAP_MAX_BYTES)
-      size = (size_t) HTS_SITEMAP_MAX_BYTES;
-    doc = body;
-  }
-  end = doc + size;
-
-  /* Set before the first callback: the handler reads the verdict. */
-  if (is_index != NULL)
-    *is_index = sitemap_root_is(doc, size, "sitemapindex");
-
-  for (p = doc; n < maxurls;) {
-    const char *loc = hts_memstr(p, (size_t) (end - p), "<loc", 4);
-    const char *val;
-    const char *stop;
-    size_t len;
-    char BIGSTK url[HTS_URLMAXSIZE];
-
-    if (loc == NULL)
-      break;
-    /* "<loc>" or "<loc xmlns:..>", never "<location>" */
-    if (loc + 4 >= end || (loc[4] != '>' && !isspace((unsigned char) loc[4]))) {
-      p = loc + 4;
-      continue;
-    }
-    val = sitemap_tag_end(loc + 4, end);
-    if (val == NULL)
-      break;
-    for (stop = val; stop < end && *stop != '<'; stop++)
-      ;
-    /* No closing tag: truncated document, so the value may be a partial URL. */
-    if (stop == end)
-      break;
-    p = stop;
-    while (val < stop && isspace((unsigned char) *val))
-      val++;
-    while (stop > val && isspace((unsigned char) *(stop - 1)))
-      stop--;
-    len = (size_t) (stop - val);
-    /* Overflow-safe: the untrusted length alone against the room left. */
-    if (len == 0 || len >= sizeof(url))
-      continue;
-    memcpy(url, val, len);
-    url[len] = '\0';
-    /* hts_unescapeEntities decodes in place and tolerates src == dest; a
-       reference to a control byte survives as one and sitemap_url_ok drops it.
-     */
-    if (hts_unescapeEntities(url, url, sizeof(url)) != 0 ||
-        !sitemap_url_ok(url))
-      continue;
-    n++;
-    if (!handler(arg, url))
-      break;
-  }
-
-  if (unpacked != NULL)
-    freet(unpacked);
-  return n;
-}
-
-/* --------------------------------------------------------------------- */
-/* Engine glue                                                           */
-/* --------------------------------------------------------------------- */
-
-static hts_sitemap_state *sitemap_get_state(httrackp *opt) {
-  if (opt->sitemap_state == NULL)
-    opt->sitemap_state = calloct(1, sizeof(hts_sitemap_state));
-  return (hts_sitemap_state *) opt->sitemap_state;
-}
-
-static sitemap_doc *sitemap_find(httrackp *opt, const char *adr,
-                                 const char *fil) {
-  hts_sitemap_state *const st = (hts_sitemap_state *) opt->sitemap_state;
-  sitemap_doc *d;
-
-  if (st == NULL)
-    return NULL;
-  for (d = st->docs; d != NULL; d = d->next) {
-    if (strfield2(d->adr, adr) && strcmp(d->fil, fil) == 0)
-      return d;
-  }
-  return NULL;
-}
-
-/* Who asked for this document decides how far it is gated. The wizard proper
-   is not usable here: it wants a referring link, and its up/down travel rules
-   would judge a child sitemap against the parent sitemap's directory. */
-static hts_boolean sitemap_fetch_allowed(httrackp *opt, const char *adr,
-                                         const char *fil,
-                                         hts_sitemap_source src) {
-  /* adr and fil are each capped just under HTS_URLMAXSIZE, and lfull prefixes
-     a scheme and a slash on top of both: 2 * HTS_URLMAXSIZE does not fit. */
-  char BIGSTK l[HTS_URLMAXSIZE * 2 + 16], lfull[HTS_URLMAXSIZE * 2 + 16];
-  int jokdepth = 0, jok;
-
-  hts_boolean refused;
-
-  /* The user naming a sitemap is the same intent as naming a start URL, which
-     the wizard admits unconditionally. */
-  if (src == HTS_SITEMAP_SRC_USER)
-    return HTS_TRUE;
-  strcpybuff(l, jump_identification_const(adr));
-  if (*fil != '/')
-    strcatbuff(l, "/");
-  strcatbuff(l, fil);
-  strcpybuff(lfull, link_has_authority(adr) ? "" : "http://");
-  strcatbuff(lfull, adr);
-  if (*fil != '/')
-    strcatbuff(lfull, "/");
-  strcatbuff(lfull, fil);
-  jok = fa_strjoker_dual(0, *opt->filters.filters, *opt->filters.filptr, lfull,
-                         l, NULL, NULL, &jokdepth);
-  refused = (jok == -1) ? HTS_TRUE : HTS_FALSE;
-  if (refused) {
-    hts_log_print(opt, LOG_NOTICE, "Sitemap: filter rule #%d refuses %s%s",
-                  jokdepth + 1, adr, fil);
-    return HTS_FALSE;
-  }
-  /* A Sitemap: line, or a sitemapindex entry, is the site inviting the fetch;
-     a Disallow elsewhere in the same file does not retract it. The well-known
-     location is only ever a guess, so there a Disallow wins. */
-  if (src == HTS_SITEMAP_SRC_GUESSED &&
-      hts_robots_forbids(opt, adr, fil, (jok != 0) ? HTS_TRUE : HTS_FALSE,
-                         refused)) {
-    hts_log_print(opt, LOG_NOTICE, "Sitemap: robots.txt forbids %s%s", adr,
-                  fil);
-    return HTS_FALSE;
-  }
-  return HTS_TRUE;
-}
-
-/* Record the link with save="" so the body stays in memory: a sitemap is
-   ingested, never mirrored. */
-static hts_boolean sitemap_queue_(httrackp *opt, const char *adr,
-                                  const char *fil, int level,
-                                  hts_sitemap_source src, hts_boolean link_it) {
-  hts_sitemap_state *const st = sitemap_get_state(opt);
-  sitemap_doc *d;
-
-  if (st == NULL)
-    return HTS_FALSE;
-  if (st->ndocs >= HTS_SITEMAP_MAX_DOCS || level > HTS_SITEMAP_MAX_LEVEL) {
-    hts_log_print(opt, LOG_WARNING, "Sitemap: cap reached, skipping %s%s", adr,
-                  fil);
-    return HTS_FALSE;
-  }
-  if (strlen(adr) >= sizeof(d->adr) || strlen(fil) >= sizeof(d->fil))
-    return HTS_FALSE;
-  if (sitemap_find(opt, adr, fil) != NULL)
-    return HTS_FALSE;
-  if (!sitemap_fetch_allowed(opt, adr, fil, src))
-    return HTS_FALSE;
-  d = calloct(1, sizeof(sitemap_doc));
-  if (d == NULL)
-    return HTS_FALSE;
-  strcpybuff(d->adr, adr);
-  strcpybuff(d->fil, fil);
-  d->level = level;
-  d->src = src;
-  d->next = st->docs;
-  st->docs = d;
-  st->ndocs++;
-
-  if (!link_it)
-    return HTS_TRUE;
-  if (!hts_record_link(opt, adr, fil, "", "", "", NULL))
-    return HTS_FALSE;
-  heap_top()->testmode = 0;
-  heap_top()->link_import = 0;
-  heap_top()->depth = opt->depth + 1;
-  heap_top()->pass2 = 0;
-  heap_top()->retry = opt->retry;
-  heap_top()->premier = heap_top_index();
-  heap_top()->precedent = heap_top_index();
-  hts_log_print(opt, LOG_INFO, "Sitemap: queued %s%s", adr, fil);
-  return HTS_TRUE;
-}
-
-static hts_boolean sitemap_queue(httrackp *opt, const char *adr,
-                                 const char *fil, int level,
-                                 hts_sitemap_source src) {
-  return sitemap_queue_(opt, adr, fil, level, src, HTS_TRUE);
-}
-
-void hts_sitemap_redirect(httrackp *opt, const char *adr, const char *fil,
-                          const char *newadr, const char *newfil) {
-  sitemap_doc *const d = sitemap_find(opt, adr, fil);
-
-  if (d == NULL || d->done)
-    return;
-  d->done = HTS_TRUE; /* the body lives at the target now */
-  /* The engine already queued the target link, so only the marking moves. */
-  (void) sitemap_queue_(opt, newadr, newfil, d->level, d->src, HTS_FALSE);
-  hts_log_print(opt, LOG_NOTICE, "Sitemap: %s%s redirects to %s%s", adr, fil,
-                newadr, newfil);
-}
-
-void hts_sitemap_seed(httrackp *opt, const char *starturl) {
-  char BIGSTK url[HTS_URLMAXSIZE * 2];
-  lien_adrfil af;
-
-  if (StringNotEmpty(opt->sitemap_url)) {
-    if (strlen(StringBuff(opt->sitemap_url)) >= sizeof(url)) {
-      hts_log_print(opt, LOG_ERROR, "Sitemap URL too long");
-    } else {
-      strcpybuff(url, StringBuff(opt->sitemap_url));
-      if (strstr(url, ":/") == NULL)
-        hts_log_print(opt, LOG_ERROR, "Sitemap URL must be absolute: %s", url);
-      else if (ident_url_absolute(url, &af) >= 0)
-        (void) sitemap_queue(opt, af.adr, af.fil, 0, HTS_SITEMAP_SRC_USER);
-    }
-  }
-  if (starturl == NULL || starturl[0] == '\0' ||
-      strlen(starturl) >= sizeof(url))
-    return;
-  strcpybuff(url, starturl);
-  if (ident_url_absolute(url, &af) < 0)
-    return;
-  {
-    hts_sitemap_state *const st = sitemap_get_state(opt);
-
-    if (st != NULL && strlen(af.adr) < sizeof(st->anchor_adr) &&
-        strlen(af.fil) < sizeof(st->anchor_fil)) {
-      strcpybuff(st->anchor_adr, af.adr);
-      strcpybuff(st->anchor_fil, af.fil);
-    }
-  }
-  if (!opt->sitemap)
-    return;
-  /* Answered in hts_sitemap_robots, once the parsed rules are installed. */
-  if (hts_record_link(opt, af.adr, "/robots.txt", "", "", "", NULL)) {
-    heap_top()->testmode = 0;
-    heap_top()->link_import = 0;
-    heap_top()->depth = 0;
-    heap_top()->pass2 = 0;
-    heap_top()->retry = opt->retry;
-    heap_top()->premier = heap_top_index();
-    heap_top()->precedent = heap_top_index();
-    /* Claim the host so the parser does not queue robots.txt a second time. */
-    if (opt->robotsptr != NULL)
-      (void) checkrobots_set((robots_wizard *) opt->robotsptr, af.adr, "");
-  }
-}
-
-void hts_sitemap_robots(httrackp *opt, const char *adr, const char *sitemaps) {
-  hts_sitemap_state *const st = (hts_sitemap_state *) opt->sitemap_state;
-  int queued = 0;
-
-  if (st == NULL || !opt->sitemap || st->probe_done ||
-      !strfield2(st->anchor_adr, adr))
-    return;
-  st->probe_done = HTS_TRUE;
-  if (sitemaps != NULL) {
-    const char *p = sitemaps;
-
-    while (*p != '\0') {
-      const char *const eol = strchr(p, '\n');
-      const size_t len = eol != NULL ? (size_t) (eol - p) : strlen(p);
-      char BIGSTK line[HTS_URLMAXSIZE];
-      lien_adrfil af;
-
-      if (len > 0 && len < sizeof(line)) {
-        memcpy(line, p, len);
-        line[len] = '\0';
-        /* Same host: a Sitemap: line must not aim the fetcher elsewhere. */
-        if (sitemap_url_ok(line) && ident_url_absolute(line, &af) >= 0 &&
-            strfield2(af.adr, adr) &&
-            sitemap_queue(opt, af.adr, af.fil, 0, HTS_SITEMAP_SRC_DECLARED))
-          queued++;
-      }
-      if (eol == NULL)
-        break;
-      p = eol + 1;
-    }
-  }
-  if (queued == 0 && !st->fallback_done) {
-    st->fallback_done = HTS_TRUE;
-    if (sitemap_queue(opt, adr, "/sitemap.xml", 0, HTS_SITEMAP_SRC_GUESSED))
-      queued++;
-  }
-  hts_log_print(opt, LOG_NOTICE, "Sitemap: %d sitemap(s) queued for %s", queued,
-                adr);
-}
-
-hts_boolean hts_sitemap_pending(httrackp *opt, const char *adr,
-                                const char *fil) {
-  const sitemap_doc *const d = sitemap_find(opt, adr, fil);
-
-  return d != NULL && !d->done ? HTS_TRUE : HTS_FALSE;
-}
-
-/* Handler context: seeding URLs from one document. */
-typedef struct sitemap_ingest_ctx {
-  httrackp *opt;
-  htsmoduleStruct *str;
-  const char *adr; /* host of the document being ingested */
-  int level;
-  hts_boolean is_index;
-  int accepted; /* URLs seeded or documents queued, not merely parsed */
-} sitemap_ingest_ctx;
-
-/* A <loc> of a <urlset>: hand it to the wizard as a top-level seed.
-   The wizard is pointed at the crawl's own start URL, not at the sitemap: the
-   site picks where its sitemap lives, so anchoring travel there would let a
-   root sitemap widen a subtree crawl to the whole host. The URL then becomes
-   its own anchor, exactly as a command-line seed does. */
-static hts_boolean sitemap_seed_url(void *arg, const char *url) {
-  sitemap_ingest_ctx *const c = (sitemap_ingest_ctx *) arg;
-  httrackp *const opt = c->opt;
-  hts_sitemap_state *const st = sitemap_get_state(opt);
-  char BIGSTK buff[HTS_URLMAXSIZE];
-  int before;
-
-  if (st == NULL || st->nurls >= HTS_SITEMAP_MAX_URLS_TOTAL) {
-    hts_log_print(opt, LOG_WARNING,
-                  "Sitemap: URL cap reached, ignoring the rest");
-    return HTS_FALSE;
-  }
-  /* strcpybuff aborts rather than truncating: never feed it unchecked input. */
-  if (strlen(url) >= sizeof(buff))
-    return HTS_TRUE;
-  st->nurls++;
-  strcpybuff(buff, url);
-  before = opt->lien_tot;
-  if (htsAddLink(c->str, buff))
-    c->accepted++;
-  if (opt->lien_tot > before)
-    heap_top()->premier = heap_top_index(); /* a seed anchors on itself */
-  return HTS_TRUE;
-}
-
-/* A <loc> of a <sitemapindex>: cross-host children are dropped, so a hostile
-   sitemap cannot aim the fetcher elsewhere. */
-static hts_boolean sitemap_seed_child(void *arg, const char *url) {
-  sitemap_ingest_ctx *const c = (sitemap_ingest_ctx *) arg;
-  char BIGSTK buff[HTS_URLMAXSIZE];
-  lien_adrfil af;
-
-  if (strlen(url) >= sizeof(buff))
-    return HTS_TRUE;
-  strcpybuff(buff, url);
-  if (ident_url_absolute(buff, &af) < 0)
-    return HTS_TRUE;
-  if (!strfield2(af.adr, c->adr)) {
-    hts_log_print(c->opt, LOG_WARNING,
-                  "Sitemap: ignoring off-host child sitemap %s%s", af.adr,
-                  af.fil);
-    return HTS_TRUE;
-  }
-  if (sitemap_queue(c->opt, af.adr, af.fil, c->level + 1,
-                    HTS_SITEMAP_SRC_DECLARED))
-    c->accepted++;
-  return HTS_TRUE;
-}
-
-/* The scan classifies the document before the first callback. */
-static hts_boolean sitemap_seed_any(void *arg, const char *url) {
-  sitemap_ingest_ctx *const c = (sitemap_ingest_ctx *) arg;
-
-  return c->is_index ? sitemap_seed_child(arg, url)
-                     : sitemap_seed_url(arg, url);
-}
-
-void hts_sitemap_ingest(httrackp *opt, htsmoduleStruct *str, const char *adr,
-                        const char *fil, const char *body, size_t size) {
-  sitemap_doc *const d = sitemap_find(opt, adr, fil);
-  hts_sitemap_state *const st = (hts_sitemap_state *) opt->sitemap_state;
-  sitemap_ingest_ctx ctx;
-  int n, anchor, saved_depth;
-
-  if (d == NULL || d->done)
-    return;
-  d->done = HTS_TRUE;
-  /* str->ptr_ is a scratch int owned by the caller, so nothing else moves. */
-  anchor = *str->ptr_;
-  if (st != NULL && st->anchor_adr[0] != '\0' && opt->hash != NULL) {
-    const int i = hash_read((const hash_struct *) opt->hash, st->anchor_adr,
-                            st->anchor_fil, 1);
-
-    if (i >= 0)
-      anchor = i;
-  }
-  *str->ptr_ = anchor;
-  /* Borrow the anchor's position but keep a seed's full depth budget. */
-  saved_depth = heap(anchor)->depth;
-  heap(anchor)->depth = opt->depth + 1;
-  ctx.opt = opt;
-  ctx.str = str;
-  ctx.adr = adr;
-  ctx.level = d->level;
-  ctx.is_index = HTS_FALSE;
-  ctx.accepted = 0;
-
-  n = hts_sitemap_scan(body, size, HTS_SITEMAP_MAX_URLS_DOC, &ctx.is_index,
-                       sitemap_seed_any, &ctx);
-  heap(anchor)->depth = saved_depth;
-  if (n < 0) {
-    hts_log_print(opt, LOG_ERROR, "Sitemap: could not decompress %s%s", adr,
-                  fil);
-    return;
-  }
-  if (ctx.is_index)
-    hts_log_print(opt, LOG_NOTICE,
-                  "Sitemap: %d of %d child sitemap(s) listed by %s%s",
-                  ctx.accepted, n, adr, fil);
-  else
-    hts_log_print(opt, LOG_NOTICE, "Sitemap: %d of %d URL(s) added from %s%s",
-                  ctx.accepted, n, adr, fil);
-}
-
-void hts_sitemap_free(httrackp *opt) {
-  hts_sitemap_state *const st = (hts_sitemap_state *) opt->sitemap_state;
-
-  if (st == NULL)
-    return;
-  while (st->docs != NULL) {
-    sitemap_doc *const next = st->docs->next;
-
-    freet(st->docs);
-    st->docs = next;
-  }
-  freet(opt->sitemap_state);
-  opt->sitemap_state = NULL;
-}

src/htssitemap.h

@@ -1,108 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* ------------------------------------------------------------ */
-/* HTTrack sitemap ingestion (sitemaps.org 0.9). Internal, not installed.
-   Reads <urlset>/<sitemapindex> documents, plain or gzip-framed, and feeds
-   their <loc> URLs to the crawl as top-level seeds. The whole input is
-   attacker-controlled, so every entry point below is capped. */
-/* ------------------------------------------------------------ */
-
-#ifndef HTS_SITEMAP_DEFH
-#define HTS_SITEMAP_DEFH
-
-#include "htsdefines.h"
-#include "htsopt.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Caps. sitemaps.org allows 50000 URLs and 50 MB uncompressed per document;
-   the byte cap sits above that so a conformant sitemap always fits. */
-#define HTS_SITEMAP_MAX_URLS_DOC 50000           /* <loc> per document */
-#define HTS_SITEMAP_MAX_URLS_TOTAL 200000        /* <loc> per mirror */
-#define HTS_SITEMAP_MAX_DOCS 256                 /* documents per mirror */
-#define HTS_SITEMAP_MAX_LEVEL 4                  /* sitemapindex nesting */
-#define HTS_SITEMAP_MAX_BYTES (64 * 1024 * 1024) /* decompressed document */
-
-/* Who asked for a sitemap document, which decides how far its fetch is gated.
-   The user naming one is the same intent as a start URL; a site declaring one
-   invites the fetch; the well-known location is only ever our guess. */
-typedef enum {
-  HTS_SITEMAP_SRC_USER,     /**< --sitemap-url */
-  HTS_SITEMAP_SRC_DECLARED, /**< a Sitemap: line or a sitemapindex entry */
-  HTS_SITEMAP_SRC_GUESSED   /**< the /sitemap.xml fallback */
-} hts_sitemap_source;
-
-/* Per-URL handler; returning HTS_FALSE stops the scan. */
-typedef hts_boolean (*hts_sitemap_handler)(void *arg, const char *url);
-
-/* Scan one sitemap document, plain or gzip-framed, handing every acceptable
-   absolute http(s) <loc> URL to `handler`. Stops after `maxurls` URLs, or when
-   the handler refuses. `is_index` (optional) reports a <sitemapindex>, whose
-   URLs are child sitemaps rather than pages. Returns the number of URLs handed
-   out, or -1 when the document could not be decompressed within the caps. */
-int hts_sitemap_scan(const char *body, size_t size, int maxurls,
-                     hts_boolean *is_index, hts_sitemap_handler handler,
-                     void *arg);
-
-/* --- Engine glue (needs a live httrackp). --- */
-
-/* Queue the first sitemap document of the mirror: the explicit --sitemap-url,
-   or the start host's /robots.txt probe for --sitemap. `starturl` is the first
-   command-line seed. No-op when neither option is set. */
-void hts_sitemap_seed(httrackp *opt, const char *starturl);
-
-/* Act on the start host's robots.txt once its rules are installed: queue the
-   Sitemap: URLs it names (newline-separated, from robots_parse), or the
-   well-known /sitemap.xml when it names none. No-op unless --sitemap. */
-void hts_sitemap_robots(httrackp *opt, const char *adr, const char *sitemaps);
-
-/* Carry the sitemap marking of (adr,fil) over to the target of a redirect the
-   engine has already queued, so a moved sitemap is still ingested. */
-void hts_sitemap_redirect(httrackp *opt, const char *adr, const char *fil,
-                          const char *newadr, const char *newfil);
-
-/* HTS_TRUE when (adr,fil) is a queued sitemap document awaiting ingestion. */
-hts_boolean hts_sitemap_pending(httrackp *opt, const char *adr,
-                                const char *fil);
-
-/* Ingest a fetched sitemap document (or the robots.txt probe): seed its URLs
-   through the wizard via htsAddLink, and queue nested sitemaps. `str` supplies
-   the parser context of the document being processed. */
-void hts_sitemap_ingest(httrackp *opt, htsmoduleStruct *str, const char *adr,
-                        const char *fil, const char *body, size_t size);
-
-/* Release the ingestion state held in opt (NULL-safe, idempotent). */
-void hts_sitemap_free(httrackp *opt);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/htsthread.c

@@ -44,18 +44,9 @@ Please visit our Website: http://www.httrack.com
 #endif
 #endif
 
-/* Outstanding threads, counted at spawn rather than by the child at entry, so
-   that a caller which spawns and immediately waits still joins them (#747). */
 static int process_chain = 0;
 static htsmutex process_chain_mutex = HTSMUTEX_INIT;
 
-static void process_chain_add(int delta) {
-  hts_mutexlock(&process_chain_mutex);
-  process_chain += delta;
-  assertf(process_chain >= 0);
-  hts_mutexrelease(&process_chain_mutex);
-}
-
 HTSEXT_API void htsthread_wait(void) {
   htsthread_wait_n(0);
 }
@@ -108,12 +99,20 @@ static void *hts_entry_point(void *tharg)
   void *const arg = s_args->arg;
   void (*fun) (void *arg) = s_args->fun;
 
-  freet(tharg);
+  free(tharg);
+
+  hts_mutexlock(&process_chain_mutex);
+  process_chain++;
+  assertf(process_chain > 0);
+  hts_mutexrelease(&process_chain_mutex);
 
   /* run */
   fun(arg);
 
-  process_chain_add(-1);
+  hts_mutexlock(&process_chain_mutex);
+  process_chain--;
+  assertf(process_chain >= 0);
+  hts_mutexrelease(&process_chain_mutex);
 #ifdef _WIN32
   return 0;
 #else
@@ -123,20 +122,18 @@ static void *hts_entry_point(void *tharg)
 
 /* create a thread */
 HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg) {
-  hts_thread_s *s_args = malloct(sizeof(hts_thread_s));
+  hts_thread_s *s_args = malloc(sizeof(hts_thread_s));
 
   assertf(s_args != NULL);
   s_args->arg = arg;
   s_args->fun = fun;
-  process_chain_add(1);
 #ifdef _WIN32
   {
     unsigned int idt;
     HANDLE handle =
       (HANDLE) _beginthreadex(NULL, 0, hts_entry_point, s_args, 0, &idt);
     if (handle == 0) {
-      process_chain_add(-1);
-      freet(s_args);
+      free(s_args);
       return -1;
     } else {
       /* detach the thread from the main process so that is can be independent */
@@ -148,23 +145,19 @@ HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg) {
     const size_t stackSize = 1024 * 1024 * 8;
     pthread_attr_t attr;
     pthread_t handle = 0;
-    hts_boolean created;
+    int retcode;
 
-    /* init kept apart: destroying an uninitialised attr is undefined (#772) */
-    if (pthread_attr_init(&attr) == 0) {
-      created = pthread_attr_setstacksize(&attr, stackSize) == 0 &&
-                pthread_create(&handle, &attr, hts_entry_point, s_args) == 0;
-      pthread_attr_destroy(&attr); /* create() copied what it needed */
-    } else {
-      created = HTS_FALSE;
-    }
-    if (!created) {
-      process_chain_add(-1);
-      freet(s_args);
+    if (pthread_attr_init(&attr) != 0
+        || pthread_attr_setstacksize(&attr, stackSize) != 0
+        || (retcode =
+            pthread_create(&handle, &attr, hts_entry_point, s_args)) != 0) {
+      free(s_args);
       return -1;
+    } else {
+      /* detach the thread from the main process so that is can be independent */
+      pthread_detach(handle);
+      pthread_attr_destroy(&attr);
     }
-    /* detach the thread from the main process so that it can be independent */
-    pthread_detach(handle);
   }
 #endif
   return 0;

src/htstools.c

@@ -276,21 +276,6 @@ int ident_url_relatif(const char *lien, const char *origin_adr,
   return ok;
 }
 
-/* Bounded substring search: bodies and archive records carry NUL bytes, so
-   strstr() would stop at the first one. */
-const char *hts_memstr(const char *hay, size_t haylen, const char *needle,
-                       size_t nlen) {
-  size_t i;
-
-  if (nlen == 0 || haylen < nlen)
-    return NULL;
-  for (i = 0; i + nlen <= haylen; i++) {
-    if (hay[i] == *needle && memcmp(hay + i, needle, nlen) == 0)
-      return hay + i;
-  }
-  return NULL;
-}
-
 // créer dans s, à partir du chemin courant curr_fil, le lien vers link (absolu)
 // un ident_url_relatif a déja été fait avant, pour que link ne soit pas un chemin relatif
 int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
@@ -324,7 +309,8 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   curr = _curr;
   strlcpybuff(curr, curr_fil, sizeof(_curr));
   if ((a = strchr(curr, '?')) == NULL) { // cut at the ? (query parameters)
-    a = hts_lastcharptr(curr);
+    // an empty path has no last character: curr-1 would read before the buffer
+    a = curr[0] != '\0' ? curr + strlen(curr) - 1 : curr;
   }
   while((*a != '/') && (a > curr))
     a--;                        // chercher dernier / du chemin courant
@@ -984,7 +970,10 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
       && toptemplate_bodycat) {
 
     strcpybuff(rpath, path);
-    hts_striplastchar(rpath, '/');
+    if (rpath[0]) {
+      if (rpath[strlen(rpath) - 1] == '/')
+        rpath[strlen(rpath) - 1] = '\0';
+    }
 
     fpo = fopen(fconcat(catbuff, sizeof(catbuff), rpath, "/index.html"), "wb");
     if (fpo) {
@@ -1197,8 +1186,11 @@ HTSEXT_API char *hts_getcategories(char *path, int type) {
   find_handle h;
   coucal hashCateg = NULL;
 
-  /* note: patching stored (inhash) value */
-  hts_striplastchar(rpath, '/');
+  if (rpath[0]) {
+    if (rpath[strlen(rpath) - 1] == '/') {
+      rpath[strlen(rpath) - 1] = '\0';  /* note: patching stored (inhash) value */
+    }
+  }
   h = hts_findfirst(rpath);
   if (h) {
     String iname = STRING_EMPTY;
@@ -1295,7 +1287,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
 
           strcpybuff(rpath, path);
           if (rpath[0]) {
-            if (hts_lastchar(rpath) != '\\')
+            if (rpath[strlen(rpath) - 1] != '\\')
               strcatbuff(rpath, "\\");
           }
           strcatbuff(rpath, "*.*");
@@ -1307,7 +1299,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
         strcpybuff(find->path, path);
         {
           if (find->path[0]) {
-            if (hts_lastchar(find->path) != '/')
+            if (find->path[strlen(find->path) - 1] != '/')
               strcatbuff(find->path, "/");
           }
         }
@@ -1436,76 +1428,3 @@ HTSEXT_API hts_boolean hts_findissystem(find_handle find) {
   }
   return 0;
 }
-
-/* Park cdst under a free sibling name; caside receives it. */
-static hts_boolean rename_park_aside(char *caside, size_t size,
-                                     const char *cdst) {
-  int i;
-
-  for (i = 0; i < 16; i++) {
-    if (!slprintfbuff(caside, size, "%s.hts-old%d", cdst, i))
-      return HTS_FALSE;
-    /* Skip a name the mirror already holds: POSIX rename() would clobber it
-       (#774). A non-regular entry reads as free and the rename refuses it. */
-    if (fexist_utf8(caside))
-      continue;
-    if (RENAME(cdst, caside) == 0)
-      return HTS_TRUE;
-  }
-  return HTS_FALSE;
-}
-
-/* cdst is in the way of the move: park it, retry, and put it back if the retry
-   fails too. Unlinking it instead would leave nothing at all (#790). */
-static hts_boolean rename_over_aside(httrackp *opt, const char *csrc,
-                                     const char *cdst) {
-  char caside[CATBUFF_SIZE];
-  int err;
-
-  /* Only a regular file may be parked: a directory in the way is not what the
-     caller asked to replace, and parking it orphans it (UNLINK cannot drop). */
-  if (!fexist_utf8(cdst))
-    return HTS_FALSE;
-  if (!rename_park_aside(caside, sizeof(caside), cdst))
-    return HTS_FALSE;
-  if (RENAME(csrc, cdst) == 0) {
-    (void) UNLINK(caside);
-    return HTS_TRUE;
-  }
-  err = errno;
-  /* Retry once, then name the parked copy: nothing else on disk or in the log
-     points at it, and an --update purge would delete it unnoticed. */
-  if (RENAME(caside, cdst) != 0 && RENAME(caside, cdst) != 0)
-    hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
-                  "could not put %s back; its previous content is now %s", cdst,
-                  caside);
-  errno = err;
-  return HTS_FALSE;
-}
-
-hts_boolean hts_rename_over(httrackp *opt, const char *src, const char *dst) {
-  char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
-
-  fconv(csrc, sizeof(csrc), src);
-  fconv(cdst, sizeof(cdst), dst);
-  if (RENAME(csrc, cdst) == 0)
-    return HTS_TRUE;
-  /* Only a dst in the way is something the fallback can clear, and the CRT maps
-     that to EEXIST; it keeps EACCES for a src another process holds, where the
-     retry would fail the same way. The src check covers a CRT that reports
-     neither. */
-  const int err = errno;
-
-  if (err != EEXIST || !fexist_utf8(src))
-    return HTS_FALSE;
-  return rename_over_aside(opt, csrc, cdst);
-}
-
-hts_boolean hts_rename_over_aside_selftest(httrackp *opt, const char *src,
-                                           const char *dst) {
-  char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
-
-  fconv(csrc, sizeof(csrc), src);
-  fconv(cdst, sizeof(cdst), dst);
-  return rename_over_aside(opt, csrc, cdst);
-}

src/htstools.h

@@ -61,11 +61,6 @@ typedef struct lien_adrfilsave lien_adrfilsave;
 int ident_url_relatif(const char *lien, const char *origin_adr,
                       const char *origin_fil,
                       lien_adrfil* const adrfil);
-/* Bounded substring search over data that may hold NUL bytes; NULL if absent.
- */
-const char *hts_memstr(const char *hay, size_t haylen, const char *needle,
-                       size_t nlen);
-
 int lienrelatif(char *s, size_t ssize, const char *link, const char *curr);
 int link_has_authority(const char *lien);
 int link_has_authorization(const char *lien);
@@ -137,18 +132,6 @@ HTSEXT_API hts_boolean hts_findisdir(find_handle find);
 HTSEXT_API hts_boolean hts_findisfile(find_handle find);
 HTSEXT_API hts_boolean hts_findissystem(find_handle find);
 
-/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both paths
-   are fconv()'d. A dst in the way is parked under a sibling name rather than
-   removed, so the old content survives a failure: back at dst, or under that
-   sibling (named in the log) when the move back failed too. Not atomic: a crash
-   between the two renames leaves dst absent and its content beside it. */
-hts_boolean hts_rename_over(httrackp *opt, const char *src, const char *dst);
-
-/* Selftest hook: run the aside fallback directly, on a platform whose rename()
-   never reaches it. Both paths are fconv()'d. */
-hts_boolean hts_rename_over_aside_selftest(httrackp *opt, const char *src,
-                                           const char *dst);
-
 #endif
 
 #endif

src/htswarc.c

@@ -38,7 +38,6 @@ Please visit our Website: http://www.httrack.com
 
 #include "htscore.h"
 #include "htslib.h"
-#include "htsback.h"
 #include "htstools.h"
 #include "htssafe.h"
 #include "htszlib.h"
@@ -58,9 +57,6 @@ Please visit our Website: http://www.httrack.com
 /* opt->state.warc value meaning "open failed once, do not retry". */
 #define WARC_DISABLED ((void *) ~(uintptr_t) 0)
 
-/* Suffix of the in-progress archive when a previous one must survive it. */
-#define WARC_TMP_SUFFIX ".tmp"
-
 struct warc_writer {
   FILE *f;
   httrackp *opt;   /* kept for close-time logging (warc_wacz_package) */
@@ -90,12 +86,6 @@ struct warc_writer {
   char *base_path;      /* resolved archive path minus .warc[.gz] suffix */
   const char *base_ext; /* ".warc.gz" or ".warc" (static) */
   char *arc_path;    /* full single-file archive path (NULL under rotation) */
-  /* A re-run must not destroy an archive it cannot replace (#759). */
-  hts_boolean protect_prev;   /* previous archive present: build in a temp */
-  hts_boolean opened;         /* open completed; a failed one swaps nothing */
-  hts_boolean failed;         /* a record or segment was lost: swap nothing */
-  uint64_t unbacked_revisits; /* URLs this pass didn't capture here: an
-                                  unbacked revisit, or nothing written at all */
   char **page_lines; /* one JSON page line per 200 text/html response, owned */
   size_t page_count;
   size_t page_cap;
@@ -129,32 +119,24 @@ static void wbuf_free(wbuf *b) {
   b->len = b->cap = 0;
 }
 
-/* Make room for n more bytes; returns 0 on success, -1 on OOM/overflow. */
-static int wbuf_reserve(wbuf *b, size_t n) {
-  size_t ncap;
-  char *nd;
-  if (n > (size_t) -1 - b->len)
-    return -1;
-  if (b->len + n <= b->cap)
-    return 0;
-  ncap = b->cap ? b->cap : 256;
-  while (ncap < b->len + n) {
-    if (ncap > (size_t) -1 / 2)
-      return -1;
-    ncap *= 2;
-  }
-  nd = realloct(b->data, ncap);
-  if (nd == NULL)
-    return -1;
-  b->data = nd;
-  b->cap = ncap;
-  return 0;
-}
-
 /* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
 static int wbuf_add(wbuf *b, const void *p, size_t n) {
-  if (wbuf_reserve(b, n) != 0)
+  if (n > (size_t) -1 - b->len)
     return -1;
+  if (b->len + n > b->cap) {
+    size_t ncap = b->cap ? b->cap : 256;
+    char *nd;
+    while (ncap < b->len + n) {
+      if (ncap > (size_t) -1 / 2)
+        return -1;
+      ncap *= 2;
+    }
+    nd = realloct(b->data, ncap);
+    if (nd == NULL)
+      return -1;
+    b->data = nd;
+    b->cap = ncap;
+  }
   memcpy(b->data + b->len, p, n);
   b->len += n;
   return 0;
@@ -166,32 +148,16 @@ static int wbuf_puts(wbuf *b, const char *s) {
 
 static int wbuf_printf(wbuf *b, const char *fmt, ...) HTS_PRINTF_FUN(2, 3);
 
-/* A header line carrying a URL has no useful bound, and giving up here loses
-   the whole record, so anything past the stack buffer is formatted into the
-   wbuf itself rather than rejected (#785). */
 static int wbuf_printf(wbuf *b, const char *fmt, ...) {
   char tmp[1024];
-  size_t need;
   int n;
   va_list ap;
   va_start(ap, fmt);
   n = vsnprintf(tmp, sizeof(tmp), fmt, ap);
   va_end(ap);
-  if (n < 0)
+  if (n < 0 || (size_t) n >= sizeof(tmp))
     return -1;
-  if ((size_t) n < sizeof(tmp))
-    return wbuf_add(b, tmp, (size_t) n);
-  need = (size_t) n + 1; /* +1: vsnprintf always writes the NUL */
-  if (wbuf_reserve(b, need) != 0)
-    return -1;
-  va_start(ap, fmt);
-  n = vsnprintf(b->data + b->len, need, fmt, ap);
-  va_end(ap);
-  /* Never advance past what was reserved, whatever the second pass returns. */
-  if (n < 0 || (size_t) n >= need)
-    return -1;
-  b->len += (size_t) n; /* the NUL is scratch, overwritten by the next append */
-  return 0;
+  return wbuf_add(b, tmp, (size_t) n);
 }
 
 /* ---- gzip-per-record member writer (mirrors ae_write_packed) ---- */
@@ -998,6 +964,16 @@ static zipFile wacz_zip_open(const char *path) {
   return zipOpen2_64(path, 0 /*create*/, NULL, &ff);
 }
 
+/* Move src onto dst (UTF-8/Windows-safe); RENAME won't clobber on Windows, so
+   fall back to unlink+rename. Returns 0 on success. */
+static int wacz_rename_over(const char *src, const char *dst) {
+  char cs[CATBUFF_SIZE], cd[CATBUFF_SIZE];
+  if (RENAME(fconv(cs, sizeof(cs), src), fconv(cd, sizeof(cd), dst)) == 0)
+    return 0;
+  (void) UNLINK(fconv(cd, sizeof(cd), dst));
+  return RENAME(fconv(cs, sizeof(cs), src), fconv(cd, sizeof(cd), dst));
+}
+
 /* Package the segment(s) + .cdx + a generated pages.jsonl into <base>.wacz at
    crawl end (the archive file(s) and .cdx are already closed on disk). */
 static void warc_wacz_package(warc_writer *w) {
@@ -1116,7 +1092,7 @@ static void warc_wacz_package(warc_writer *w) {
     hts_log_print(w->opt, LOG_WARNING,
                   "WACZ: packaging failed, kept existing %s untouched",
                   waczpath);
-  } else if (!hts_rename_over(w->opt, tmppath, waczpath)) {
+  } else if (wacz_rename_over(tmppath, waczpath) != 0) {
     (void) UNLINK(fconv(catbuff, sizeof(catbuff), tmppath));
     hts_log_print(w->opt, LOG_WARNING | LOG_ERRNO,
                   "WACZ: could not finalize %s", waczpath);
@@ -1163,17 +1139,13 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
      never split a record, and never rotate a warcinfo (it opens a segment). */
   if (w->max_size > 0 && w->seg_base != NULL && w->offset >= w->max_size &&
       strcmp(type, "warcinfo") != 0) {
-    if (warc_rotate(w) != 0) {
-      w->failed = HTS_TRUE;
+    if (warc_rotate(w) != 0)
       return -1;
-    }
   }
 
   /* F4: overflow-safe block length; http_hdr_len+sep is provably small. */
-  if (payload > (size_t) -1 - http_hdr_len - sep) {
-    w->failed = HTS_TRUE;
+  if (payload > (size_t) -1 - http_hdr_len - sep)
     return -1;
-  }
   block_len = http_hdr_len + sep + payload;
 
   memset(&hdr, 0, sizeof(hdr));
@@ -1279,23 +1251,11 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
   rc = 0;
 done:
   wbuf_free(&hdr);
-  if (rc != 0)
-    w->failed = HTS_TRUE; /* a truncated run must not replace a whole one */
   return rc;
 }
 
 /* ---- segment rotation (--warc-max-size) ---- */
 
-/* Path to open for the segment whose final path is `final`: that path itself,
-   or a sibling temp while a previous archive must survive until close. */
-static const char *warc_open_path(warc_writer *w, const char *final, char *buf,
-                                  size_t bufsz) {
-  if (!w->protect_prev)
-    return final;
-  snprintf(buf, bufsz, "%s" WARC_TMP_SUFFIX, final);
-  return buf;
-}
-
 /* Emit the warcinfo that heads a segment; sets w->info_id for its records. */
 static int warc_write_warcinfo_record(warc_writer *w) {
   w->info_id[0] = '\0'; /* warcinfo itself carries no WARC-Warcinfo-ID */
@@ -1307,23 +1267,17 @@ static int warc_write_warcinfo_record(warc_writer *w) {
 
 static int warc_rotate(warc_writer *w) {
   char namebuf[HTS_URLMAXSIZE * 2];
-  char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
   char catbuff[CATBUFF_SIZE];
-  const unsigned next = w->seg + 1;
-  FILE *f;
-  snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, next,
-           w->seg_ext);
-  /* Open before advancing: w->seg must only ever name a segment that exists,
-     or close-time packaging and swapping would work on a missing file. */
-  f = FOPEN(fconv(catbuff, sizeof(catbuff),
-                  warc_open_path(w, namebuf, openbuf, sizeof(openbuf))),
-            "wb");
-  if (f == NULL)
-    return -1;
-  if (w->f != NULL)
+  if (w->f != NULL) {
     fclose(w->f);
-  w->f = f;
-  w->seg = next;
+    w->f = NULL;
+  }
+  w->seg++;
+  snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
+           w->seg_ext);
+  w->f = FOPEN(fconv(catbuff, sizeof(catbuff), namebuf), "wb");
+  if (w->f == NULL)
+    return -1;
   w->offset = 0;
   if (w->cdx_on) {
     freet(w->cur_seg);
@@ -1356,24 +1310,12 @@ void warc_free_request(htsblk *r) {
     freet(r->warc_resphdr);
     if (r->warc_rawpath != NULL) {
       (void) UNLINK(r->warc_rawpath); /* owns the verbatim spool file */
-      back_tmpdir_drop(r->warc_rawpath);
       freet(r->warc_rawpath);
       r->warc_rawpath = NULL;
     }
   }
 }
 
-void warc_move_request(htsblk *src, htsblk *dst) {
-  if (src == NULL || dst == NULL || src == dst)
-    return;
-  freet(dst->warc_reqhdr);
-  freet(dst->warc_resphdr);
-  dst->warc_reqhdr = src->warc_reqhdr;
-  dst->warc_resphdr = src->warc_resphdr;
-  src->warc_reqhdr = NULL;
-  src->warc_resphdr = NULL;
-}
-
 void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path) {
   if (r != NULL) {
     LLint rawsize;
@@ -1392,7 +1334,6 @@ void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path) {
 warc_writer *warc_open(httrackp *opt, const char *path) {
   warc_writer *w;
   char namebuf[HTS_URLMAXSIZE * 2];
-  char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
   char catbuff[CATBUFF_SIZE];
   wbuf info;
   const char *robots;
@@ -1406,8 +1347,16 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
     char ts[32];
     time_t t = time(NULL);
     struct tm tmv;
-    if (!hts_gmtime(t, &tmv))
+#if defined(_WIN32)
+    struct tm *g = gmtime(&t);
+    if (g != NULL)
+      tmv = *g;
+    else
       memset(&tmv, 0, sizeof(tmv));
+#else
+    if (gmtime_r(&t, &tmv) == NULL)
+      memset(&tmv, 0, sizeof(tmv));
+#endif
     strftime(ts, sizeof(ts), "%Y%m%d%H%M%S", &tmv);
     snprintf(catbuff, sizeof(catbuff), "httrack-%s.warc.gz", ts);
     path =
@@ -1535,105 +1484,35 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
     path = namebuf;
   }
 
-  if (w->max_size == 0 && (w->arc_path = strdupt(path)) == NULL) {
-    warc_close(w);
-    return NULL;
-  }
-  /* Set only once arc_path is recorded, so a half-built writer never swaps. */
-  w->protect_prev = fsize_utf8(path) > 0 ? HTS_TRUE : HTS_FALSE;
-  w->f = FOPEN(fconv(catbuff, sizeof(catbuff),
-                     warc_open_path(w, path, openbuf, sizeof(openbuf))),
-               "wb");
+  w->f = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
   if (w->f == NULL) {
     warc_close(w);
     return NULL;
   }
   if (w->cdx_on)
     w->cur_seg = path_basename_dup(path);
+  if (w->wacz_on && w->max_size == 0)
+    w->arc_path = strdupt(path); /* single-file: package this exact path */
 
   if (warc_write_warcinfo_record(w) != 0) {
     warc_close(w);
     return NULL;
   }
-  w->opened = HTS_TRUE;
   return w;
 }
 
-/* Final path of segment s (the run's only archive when rotation is off). */
-static const char *warc_seg_path(warc_writer *w, unsigned s, char *buf,
-                                 size_t bufsz) {
-  if (w->max_size == 0)
-    return w->arc_path;
-  snprintf(buf, bufsz, "%s-%05u%s", w->seg_base, s, w->seg_ext);
-  return buf;
-}
-
-/* Swap this run's archive into place, unless it only holds revisits naming
-   bodies the previous archive still has and this one does not (#759).
-   HTS_FALSE: the previous archive was kept, so leave its .cdx and .wacz too. */
-static hts_boolean warc_commit(warc_writer *w) {
-  char finalbuf[HTS_URLMAXSIZE * 2];
-  char tmpbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
-  char catbuff[CATBUFF_SIZE];
-  const unsigned nseg = (w->max_size > 0) ? w->seg + 1 : 1;
-  hts_boolean swap;
-  unsigned s;
-
-  if (!w->protect_prev)
-    return HTS_TRUE; /* nothing was there to lose: written in place */
-
-  /* All or nothing: a swap stopping halfway would mix this run's segments with
-     the previous one's, so require every temp before renaming any. */
-  swap = w->opened && !w->failed && w->unbacked_revisits == 0;
-  for (s = 0; s < nseg && swap; s++) {
-    snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
-             warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
-    swap = fsize_utf8(tmpbuf) > 0;
-  }
-
-  if (swap) {
-    for (s = 0; s < nseg; s++) {
-      const char *final = warc_seg_path(w, s, finalbuf, sizeof(finalbuf));
-      snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX, final);
-      if (!hts_rename_over(w->opt, tmpbuf, final)) {
-        hts_log_print(w->opt, LOG_ERROR | LOG_ERRNO,
-                      "WARC: could not replace %s", final);
-        return HTS_FALSE;
-      }
-    }
-    return HTS_TRUE;
-  }
-
-  for (s = 0; s < nseg; s++) {
-    snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
-             warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
-    (void) UNLINK(fconv(catbuff, sizeof(catbuff), tmpbuf));
-  }
-  if (w->unbacked_revisits > 0)
-    hts_log_print(
-        w->opt, LOG_ERROR,
-        "WARC: this pass revisited %llu URL(s) without re-downloading them, so "
-        "this archive doesn't hold their current content; kept the previous %s "
-        "(re-run with -C0, or --warc-file with a name of its own)",
-        (unsigned long long) w->unbacked_revisits,
-        warc_seg_path(w, 0, finalbuf, sizeof(finalbuf)));
-  return HTS_FALSE;
-}
-
 void warc_close(warc_writer *w) {
   size_t i;
   if (w == NULL)
     return;
+  warc_cdx_flush(w); /* sort + write <base>.cdx before tearing down */
   if (w->f != NULL)
     fclose(w->f);
   w->f = NULL;
-  if (warc_commit(w)) {
-    warc_cdx_flush(w); /* sort + write <base>.cdx beside the archive */
 #if HTS_USEOPENSSL
-    if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
-      warc_wacz_package(w);
+  if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
+    warc_wacz_package(w);
 #endif
-  }
   if (w->seen != NULL)
     coucal_delete(&w->seen);
   for (i = 0; i < w->cdx_count; i++)
@@ -1678,8 +1557,8 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
                            const char *ip, const char *req_hdr,
                            const char *resp_hdr, const char *body,
                            size_t body_len, const char *body_path,
-                           const char *content_type, int statuscode,
-                           int unchanged_kind, int truncated) {
+                           int statuscode, int is_update_unchanged,
+                           int truncated) {
   wbuf http;
   char resp_id[64];
   char pdig[33];
@@ -1702,43 +1581,18 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
    */
   snprintf(statusbuf, sizeof(statusbuf), "%d", statuscode);
   http_header_value(resp_hdr, "Content-Type", mimebuf, sizeof(mimebuf));
-  /* a 304 declares no type, so index the one only the caller knows */
-  if (mimebuf[0] == '\0' && content_type != NULL)
-    strlncatbuff(mimebuf, content_type, sizeof(mimebuf), sizeof(mimebuf) - 1);
 
   /* A payload exists (for digesting) unless this is a bodyless 304. */
   has_payload = (body_len > 0 && (body != NULL || body_path != NULL) &&
-                 unchanged_kind != WARC_UNCHANGED_SERVER_304);
+                 !is_update_unchanged);
 
   /* Payload digest drives identical-payload-digest dedup (OpenSSL only). */
   have_pdig =
       has_payload ? payload_digest_b32(body, body_len, body_path, pdig) : 0;
 
-  if (unchanged_kind == WARC_UNCHANGED_SERVER_304) {
+  if (is_update_unchanged) {
     is_revisit = 1;
     profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
-    /* Replay resolves a revisit by this field alone, and a 304 stands in for
-       the same URL. No WARC-Refers-To-Date to go with it: the cache keeps the
-       document's Last-Modified, never the previous capture time. */
-    refers_uri = target_uri;
-    /* Served from cache: the payload sits in the previous archive, not here. */
-    w->unbacked_revisits++;
-  } else if (unchanged_kind == WARC_UNCHANGED_ENGINE_FORCED) {
-    /* has_payload requires body_len>0, so a genuinely empty body looks
-       digest-less too; it still has a well-defined digest (sha1 of nothing),
-       so compute it here rather than treat it as a missing-crypto case. */
-    if (!have_pdig && body_len == 0 && (body != NULL || body_path != NULL))
-      have_pdig = payload_digest_b32(body, body_len, body_path, pdig);
-    /* Served from cache with no exchange either way: still unbacked. */
-    w->unbacked_revisits++;
-    /* No digest (no OpenSSL) means nothing to point a revisit at, and there
-       was no real exchange to record as a response; write nothing (#839). */
-    if (!have_pdig)
-      return 0;
-    is_revisit = 1;
-    profile =
-        "http://netpreserve.org/warc/1.1/revisit/identical-payload-digest";
-    refers_uri = target_uri;
   } else if (have_pdig && w->seen != NULL) {
     void *prev = NULL;
     if (coucal_read_pvoid(w->seen, pdig, &prev) && prev != NULL) {
@@ -1848,7 +1702,7 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
   const char *body_path;
   const char *resp_hdr;
   char synth[512];
-  int unchanged_kind;
+  int is_unchanged;
   int is_ftp;
 
   if (opt->state.warc == WARC_DISABLED)
@@ -1909,12 +1763,7 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
     body_len = (size_t) back->r.warc_rawsize;
   }
 
-  if (!back->r.notmodified || !opt->is_update)
-    unchanged_kind = WARC_UNCHANGED_NONE;
-  else if (back->r.warc_forced_notmodified)
-    unchanged_kind = WARC_UNCHANGED_ENGINE_FORCED;
-  else
-    unchanged_kind = WARC_UNCHANGED_SERVER_304;
+  is_unchanged = (back->r.notmodified && opt->is_update) ? 1 : 0;
 
   /* Prefer the stashed raw headers; synthesize a minimal status line for the
      header-less (HTTP/0.9-style) responses that never carried a header block.
@@ -1929,7 +1778,6 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
   }
 
   warc_write_transaction(w, uri, ip, back->r.warc_reqhdr, resp_hdr, body,
-                         body_len, body_path, back->r.contenttype,
-                         back->r.statuscode, unchanged_kind,
+                         body_len, body_path, back->r.statuscode, is_unchanged,
                          back->r.warc_truncated);
 }

src/htswarc.h

@@ -68,10 +68,6 @@ void warc_stash_response(htsblk *r, const char *resphdr);
 /* Free both stashed header blocks (idempotent, NULL-safe). */
 void warc_free_request(htsblk *r);
 
-/* Move both stashed header blocks from src to dst, freeing dst's own. For the
-   304 path, which replaces the whole htsblk with the cache entry. */
-void warc_move_request(htsblk *src, htsblk *dst);
-
 /* Adopt the de-chunked compressed spool at tmpfile_path onto
    r->warc_rawpath/warc_rawsize (strdupt; frees any prior) so the WARC record
    stores the body verbatim. No-op leaving warc_rawpath NULL when tmpfile_path
@@ -101,12 +97,6 @@ void warc_close(warc_writer *w);
    success, -1 on error or truncation. Exposed for the -#test=warc-surt test. */
 int warc_surt(const char *url, char *out, size_t outsz);
 
-/* unchanged_kind for warc_write_transaction. */
-#define WARC_UNCHANGED_NONE 0       /* a fresh exchange: a full response */
-#define WARC_UNCHANGED_SERVER_304 1 /* the server itself answered 304 */
-#define WARC_UNCHANGED_ENGINE_FORCED                                           \
-  2 /* the engine, not the server, decided nothing changed (#839) */
-
 /* Write one transaction's request + response (or revisit) records.
    target_uri:  absolute URL fetched.
    ip:          numeric peer IP, or NULL/"" to omit.
@@ -114,11 +104,7 @@ int warc_surt(const char *url, char *out, size_t outsz);
    resp_hdr:    raw received response header block (status line + headers).
    body/body_len: decoded in-memory body, or NULL when on disk.
    body_path:   file re-read for the body when body==NULL (may be NULL).
-   content_type: CDXJ mime when resp_hdr declares none, as a 304 does; NULL
-                to leave the index line without one.
-   unchanged_kind: WARC_UNCHANGED_*; SERVER_304 writes a server-not-modified
-                revisit, ENGINE_FORCED a self-referencing identical-payload-
-                digest one (no 304 to claim), both unbacked (#839).
+   is_update_unchanged: nonzero for a 304 server-not-modified revisit.
    truncated: a WARC_TRUNC_* reason to tag a cap-truncated body, else 0.
    The body is stored verbatim: Content-Encoding is kept and Content-Length set
    to body_len, so body/body_len must be the as-received (coded) bytes.
@@ -127,8 +113,8 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
                            const char *ip, const char *req_hdr,
                            const char *resp_hdr, const char *body,
                            size_t body_len, const char *body_path,
-                           const char *content_type, int statuscode,
-                           int unchanged_kind, int truncated);
+                           int statuscode, int is_update_unchanged,
+                           int truncated);
 
 /* Write one non-HTTP capture as a single WARC 'resource' record: the block is
    the raw payload (no HTTP envelope), Content-Type is the payload's own MIME.

src/htsweb.c

@@ -101,8 +101,8 @@ static void htsweb_sig_brpipe(int code) {
   /* ignore */
 }
 
-/* Threads that never return; no wait may count on them draining. */
-static int nonjoinable_threads = 0;
+/* Number of background threads */
+static int background_threads = 0;
 
 /* Server/client ping handling */
 static htsmutex pingMutex = HTSMUTEX_INIT;
@@ -224,7 +224,9 @@ int main(int argc, char *argv[]) {
 #ifdef HTS_USESWF
   smallserver_setkey("USESWF", "1");
 #endif
+#ifdef HTS_USEZLIB
   smallserver_setkey("USEZLIB", "1");
+#endif
 #ifdef _WIN32
   smallserver_setkey("WIN32", "1");
 #endif
@@ -297,19 +299,15 @@ int main(int argc, char *argv[]) {
 
   /* pinger */
   if (parentPid > 0) {
-    if (hts_newthread(client_ping, (void *) (uintptr_t) parentPid) == 0) {
-#ifndef _WIN32
-      nonjoinable_threads++; /* client_ping() only ever leaves through exit() */
-#endif
-    }
+    hts_newthread(client_ping, (void *) (uintptr_t) parentPid);
+    background_threads++; /* Do not wait for this thread! */
     smallserver_setpinghandler(pingHandler, NULL);
   }
 
   /* launch */
   ret = help_server(argv[1], defaultPort, bindAddr);
 
-  /* Drain everything a mirror may still have in flight, the pinger aside. */
-  htsthread_wait_n(nonjoinable_threads);
+  htsthread_wait_n(background_threads - 1);
   hts_uninit();
 
 #ifdef _WIN32
@@ -384,6 +382,7 @@ void webhttrack_main(char *cmd) {
   commandRunning = 1;
   DEBUG(fprintf(stderr, "commandRunning=1\n"));
   hts_newthread(back_launch_cmd, (void *) strdup(cmd));
+  background_threads++; /* Do not wait for this thread! */
 }
 
 void webhttrack_lock(void) {
@@ -424,8 +423,8 @@ static int webhttrack_runmain(httrackp * opt, int argc, char **argv) {
   /* Rock'in! */
   ret = hts_main2(argc, argv, opt);
 
-  /* Wait for pending threads to finish; the pinger and this thread stay. */
-  htsthread_wait_n(nonjoinable_threads + 1);
+  /* Wait for pending threads to finish */
+  htsthread_wait_n(background_threads);
 
   return ret;
 }

src/htswizard.c

@@ -151,23 +151,6 @@ static hts_boolean is_embed_pair(const htspair_t *table, const char *tag,
   return HTS_FALSE;
 }
 
-/* The engine's robots.txt verdict for (adr,fil). Under HTS_ROBOTS_SOMETIMES an
-   explicit filter acceptance overrides the ban, which is why the filter outcome
-   is an input; the sitemap fetcher asks the same question outside the wizard.
- */
-hts_boolean hts_robots_forbids(httrackp *opt, const char *adr, const char *fil,
-                               hts_boolean filters_decided,
-                               hts_boolean filters_refused) {
-  if (!opt->robots || opt->robotsptr == NULL)
-    return HTS_FALSE;
-  if (checkrobots((robots_wizard *) opt->robotsptr, adr, fil) != -1)
-    return HTS_FALSE;
-  if (filters_decided && !filters_refused &&
-      opt->robots == HTS_ROBOTS_SOMETIMES)
-    return HTS_FALSE;
-  return HTS_TRUE;
-}
-
 static int hts_acceptlink_(httrackp * opt, int ptr,
                            const char *adr, const char *fil, const char *tag,
                            const char *attribute, int *set_prio_to,
@@ -374,8 +357,8 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
         forbidden_url = 1;
       break;                    // interdicton de sortir au dela de l'adresse
     case HTS_TRAVEL_SAME_DOMAIN: {
-      size_t i = hts_lastcharoffset(adr);
-      size_t j = hts_lastcharoffset(urladr());
+      size_t i = strlen(adr) - 1;
+      size_t j = strlen(urladr()) - 1;
 
       if ((i > 0) && (j > 0)) {
         while ((i > 0) && (adr[i] != '.'))
@@ -411,8 +394,8 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
         forbidden_url = 1;
     } break;
     case HTS_TRAVEL_SAME_TLD: {
-      size_t i = hts_lastcharoffset(adr);
-      size_t j = hts_lastcharoffset(urladr());
+      size_t i = strlen(adr) - 1;
+      size_t j = strlen(urladr()) - 1;
 
       while ((i > 0) && (adr[i] != '.'))
         i--;
@@ -593,26 +576,30 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
       }
     }
     // vérifier robots.txt
-    if (opt->robots && checkrobots(_ROBOTS, adr, fil) == -1) {
+    if (opt->robots) {
+      int r = checkrobots(_ROBOTS, adr, fil);
+
+      if (r == -1) {            // interdiction
 #if DEBUG_ROBOTS
-      printf("robots.txt forbidden: %s%s\n", adr, fil);
+        printf("robots.txt forbidden: %s%s\n", adr, fil);
 #endif
-      if (!hts_robots_forbids(opt, adr, fil,
-                              (!question && filters_answer) ? HTS_TRUE
-                                                            : HTS_FALSE,
-                              (forbidden_url == 1) ? HTS_TRUE : HTS_FALSE)) {
-        if (!forbidden_url) {
-          hts_log_print(
-              opt, LOG_DEBUG,
-              "Warning link followed against robots.txt: link %s at %s%s", l,
-              adr, fil);
+        // question résolue, par les filtres, et mode robot non strict
+        if ((!question) && (filters_answer) &&
+            (opt->robots == HTS_ROBOTS_SOMETIMES) && (forbidden_url != 1)) {
+          r = 0;                // annuler interdiction des robots
+          if (!forbidden_url) {
+            hts_log_print(opt, LOG_DEBUG,
+                          "Warning link followed against robots.txt: link %s at %s%s",
+                          l, adr, fil);
+          }
+        }
+        if (r == -1) {          // interdire
+          forbidden_url = 1;
+          question = 0;
+          hts_log_print(opt, LOG_DEBUG,
+                        "(robots.txt) forbidden link: link %s at %s%s", l, adr,
+                        fil);
         }
-      } else {
-        forbidden_url = 1;
-        question = 0;
-        hts_log_print(opt, LOG_DEBUG,
-                      "(robots.txt) forbidden link: link %s at %s%s", l, adr,
-                      fil);
       }
     }
 
@@ -728,7 +715,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
       case 1: // forbid the whole directory and subdirs: adr/path/*
         forbidden_url = 1;
         {
-          size_t i = hts_lastcharoffset(fil);
+          size_t i = strlen(fil) - 1;
 
           while((fil[i] != '/') && (i > 0))
             i--;
@@ -794,7 +781,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
       case 5: // allow the whole directory and its children
         if ((opt->seeker & HTS_SEEKER_UP) == 0) { // not allowed to go up
-          size_t i = hts_lastcharoffset(fil);
+          size_t i = strlen(fil) - 1;
 
           while((fil[i] != '/') && (i > 0))
             i--;
@@ -836,7 +823,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
         //
       case 7: // allow this directory
       {
-        size_t i = hts_lastcharoffset(fil);
+        size_t i = strlen(fil) - 1;
 
         while ((fil[i] != '/') && (i > 0))
           i--;

src/htswizard.h

@@ -49,13 +49,6 @@ typedef struct httrackp httrackp;
 typedef struct lien_url lien_url;
 #endif
 
-/* The engine's robots.txt verdict for (adr,fil): HTS_TRUE when the fetch is
-   forbidden. `filters_decided`/`filters_refused` carry the filter outcome,
-   which overrides a ban under -s1 (HTS_ROBOTS_SOMETIMES). */
-hts_boolean hts_robots_forbids(httrackp *opt, const char *adr, const char *fil,
-                               hts_boolean filters_decided,
-                               hts_boolean filters_refused);
-
 int hts_acceptlink(httrackp * opt, int ptr,
                    const char *adr, const char *fil,
                    const char *tag, const char *attribute,

src/htszlib.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscodec.h"
 #include "htszlib.h"
 
+#if HTS_USEZLIB
 /* zlib */
 /*
 #include <zlib.h>
@@ -273,3 +274,4 @@ const char *hts_get_zerror(int err) {
     break;
   }
 }
+#endif

src/httrack.c

@@ -306,8 +306,8 @@ int main(int argc, char **argv) {
     fprintf(stderr, "* %s\n", hts_errmsg(opt));
   }
   global_opt = NULL;
-  htsthread_wait(); /* pending threads still read opt */
   hts_free_opt(opt);
+  htsthread_wait();             /* wait for pending threads */
   hts_uninit();
 
 #ifdef _WIN32

src/libhttrack.vcxproj

@@ -140,7 +140,6 @@
     <ClCompile Include="htszlib.c" />
     <ClCompile Include="htswarc.c" />
     <ClCompile Include="htschanges.c" />
-    <ClCompile Include="htssitemap.c" />
     <ClCompile Include="md5.c" />
     <ClCompile Include="minizip\ioapi.c" />
     <ClCompile Include="minizip\iowin32.c" />

src/proxy/proxytrack.c

@@ -520,13 +520,12 @@ static void proxytrack_add_DAV_Item(String * item, String * buff,
                                     const char *filename, size_t size,
                                     time_t timestamp, const char *mime,
                                     int isDir, int isRoot, int isDefault) {
-  struct tm timetmbuf;
-  struct tm *timetm = &timetmbuf;
+  struct tm *timetm;
 
   if (timestamp == (time_t) 0 || timestamp == (time_t) - 1) {
     timestamp = time(NULL);
   }
-  if (hts_gmtime(timestamp, timetm)) {
+  if ((timetm = gmtime(&timestamp)) != NULL) {
     char tms[256 + 1];
     const char *name;
 
@@ -748,8 +747,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
             const char *thisUrl = list[i];
             const char *mimeType = "application/octet-stream";
             unsigned int thisUrlLen = (unsigned int) strlen(thisUrl);
-            /* the folder's default document is enumerated as an empty name */
-            int thisIsDir = (hts_lastchar(thisUrl) == '/') ? 1 : 0;
+            int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0;
 
             /* Item URL */
             StringRoom(itemUrl,
@@ -854,7 +852,8 @@ static PT_Element proxytrack_process_HTTP_List(PT_Indexes indexes,
     for(isDir = 1; isDir >= 0; isDir--) {
       for(i = 0; list[i] != NULL; i++) {
         char *thisUrl = list[i];
-        int thisIsDir = (hts_lastchar(thisUrl) == '/') ? 1 : 0;
+        unsigned int thisUrlLen = (unsigned int) strlen(thisUrl);
+        int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0;
 
         if (thisIsDir == isDir) {
           if (isDir)

src/proxy/proxytrack.h

@@ -360,14 +360,21 @@ HTS_UNUSED static struct tm *convert_time_rfc822(struct tm *result, const char *
 HTS_UNUSED static struct tm PT_GetTime(time_t t) {
   struct tm tmbuf;
 
-  if (!hts_gmtime(t, &tmbuf)) {
+#ifdef _WIN32
+  struct tm *tm = gmtime(&t);
+#else
+  struct tm *tm = gmtime_r(&t, &tmbuf);
+#endif
+  if (tm != NULL)
+    return *tm;
+  else {
     /* an all-zero tm has tm_mday == 0, which the ARC date field prints as a
        day of "00"; the epoch is the conventional "date unknown" */
     memset(&tmbuf, 0, sizeof(tmbuf));
     tmbuf.tm_year = 70;
     tmbuf.tm_mday = 1;
+    return tmbuf;
   }
-  return tmbuf;
 }
 HTS_UNUSED static int set_filetime(const char *file, struct tm *tm_time) {
   struct utimbuf tim;

src/proxy/store.c

@@ -259,6 +259,7 @@ static int binput(char *buff, char *s, int max) {
   int count = 0;
   int destCount = 0;
 
+  // Note: \0 will return 1
   while(destCount < max && buff[count] != '\0' && buff[count] != '\n') {
     if (buff[count] != '\r') {
       s[destCount++] = buff[count];
@@ -267,9 +268,8 @@ static int binput(char *buff, char *s, int max) {
   }
   s[destCount] = '\0';
 
-  /* only step over a real separator: past the terminating NUL leaves the
-     caller's buffer, past a truncated field's next byte silently eats it */
-  return buff[count] == '\n' ? count + 1 : count;
+  // then return the supplemental jump offset
+  return count + 1;
 }
 
 static time_t file_timestamp(const char *file) {
@@ -888,31 +888,6 @@ static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags)
 	} \
 } while(0)
 
-/* Set path (capacity size) to filename's parent directory, separator included,
-   from an absolute filename. Empty when there is none, or when it would not
-   fit: a truncated prefix names a different directory. */
-static void index_base_path(char *path, size_t size, const char *filename) {
-  const char *abpath;
-  int slashes;
-
-  for (slashes = 2, abpath = hts_lastcharptr(filename);
-       abpath > filename &&
-       ((*abpath != '/' && *abpath != '\\') || --slashes > 0);
-       abpath--)
-    ;
-  path[0] = '\0';
-  if (slashes == 0 && *abpath != 0 && (size_t) (abpath - filename) < size - 1) {
-    int i;
-
-    strlncatbuff(path, filename, size, (size_t) (abpath - filename) + 1);
-    for (i = 0; path[i] != 0; i++) {
-      if (path[i] == '\\') {
-        path[i] = '/';
-      }
-    }
-  }
-}
-
 int PT_LoadCache__New(PT_Index index_, const char *filename) {
   if (index_ != NULL && filename != NULL) {
     PT_Index__New index = &index_->slots.formatNew;
@@ -923,9 +898,26 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
 
     // Opened ?
     if (zFile != NULL) {
+      const char *abpath;
+      int slashes;
       coucal hashtable = index->hash;
 
-      index_base_path(index->path, sizeof(index->path), filename);
+      /* Compute base path for this index - the filename MUST be absolute! */
+      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
+          abpath > filename && ((*abpath != '/' && *abpath != '\\')
+                                || --slashes > 0);
+          abpath--) ;
+      index->path[0] = '\0';
+      if (slashes == 0 && *abpath != 0) {
+        int i;
+
+        strncat(index->path, filename, (int) (abpath - filename) + 1);
+        for(i = 0; index->path[i] != 0; i++) {
+          if (index->path[i] == '\\') {
+            index->path[i] = '/';
+          }
+        }
+      }
 
       /* Ready directory entries */
       if (unzGoToFirstFile(zFile) == Z_OK) {
@@ -1390,10 +1382,11 @@ static int cache_brstr(char *adr, char *s, size_t s_size) {
   char buff[256 + 4];
 
   off = binput(adr, buff, 256);
-  /* no length-prefixed value follows a field the terminating NUL stopped */
-  if (adr[off] == '\0') {
+  /* binput stops at adr's terminating NUL; a value only follows a real line
+     terminator, so never step past end-of-buffer. */
+  if (adr[off - 1] == '\0') {
     s[0] = '\0';
-    return off;
+    return off - 1;
   }
   /* an empty/non-numeric field leaves i unset: treat as length 0 */
   if (sscanf(buff, "%d", &i) != 1 || i < 0 || i > 32768)
@@ -1497,23 +1490,41 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
 
     cache->filenameDat[0] = '\0';
     cache->filenameNdx[0] = '\0';
+    cache->path[0] = '\0';
 
-    index_base_path(cache->path, sizeof(cache->path), filename);
+    {
+      PT_Index__Old index = cache;
+      const char *abpath;
+      int slashes;
+
+      /* -------------------- COPY OF THE __New() CODE -------------------- */
+      /* Compute base path for this index - the filename MUST be absolute! */
+      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
+          abpath > filename && ((*abpath != '/' && *abpath != '\\')
+                                || --slashes > 0);
+          abpath--) ;
+      index->path[0] = '\0';
+      if (slashes == 0 && *abpath != 0) {
+        int i;
+
+        strncat(index->path, filename, (int) (abpath - filename) + 1);
+        for(i = 0; index->path[i] != 0; i++) {
+          if (index->path[i] == '\\') {
+            index->path[i] = '/';
+          }
+        }
+      }
+      /* -------------------- END OF COPY OF THE __New() CODE -------------------- */
+    }
 
     /* Index/data filenames */
     if (pos != NULL) {
-      const size_t nLen = (size_t) (pos - filename);
+      int nLen = (int) (pos - filename);
 
-      /* a base clipped to fit would name a different pair of files */
-      if (nLen > sizeof(cache->filenameDat) - sizeof(".dat")) {
-        return 0;
-      }
-      strlncatbuff(cache->filenameDat, filename, sizeof(cache->filenameDat),
-                   nLen);
-      strlncatbuff(cache->filenameNdx, filename, sizeof(cache->filenameNdx),
-                   nLen);
-      strcatbuff(cache->filenameDat, ".dat");
-      strcatbuff(cache->filenameNdx, ".ndx");
+      strncat(cache->filenameDat, filename, nLen);
+      strncat(cache->filenameNdx, filename, nLen);
+      strcat(cache->filenameDat, ".dat");
+      strcat(cache->filenameNdx, ".ndx");
     }
     ndxSize = filesize(cache->filenameNdx);
     cache->timestamp = file_timestamp(cache->filenameDat);
@@ -1576,10 +1587,7 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
               }
               /* read position */
               a += binput(a, linepos, 200);
-              /* an unparseable field must not carry the previous entry's
-                 offset over, nor read the stack on the first one */
-              if (sscanf(linepos, "%d", &pos) != 1)
-                pos = 0;
+              sscanf(linepos, "%d", &pos);
 
               /* Add entry */
               coucal_add(cache->hash, line, pos);
@@ -2014,12 +2022,7 @@ static time_t getGMT(struct tm *tm) {   /* hey, time_t is local! */
     /* BSD does not have static "timezone" declared */
 #if (defined(BSD) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__FreeBSD_kernel__))
     time_t now = time(NULL);
-    struct tm nowtm;
-    time_t timezone;
-
-    if (!hts_localtime(now, &nowtm))
-      return (time_t) -1;
-    timezone = -nowtm.tm_gmtoff;
+    time_t timezone = -localtime(&now)->tm_gmtoff;
 #elif defined(_MSC_VER)
     /* MSVC spells it _timezone */
     const time_t timezone = _timezone;
@@ -2369,28 +2372,11 @@ typedef struct PT_SaveCache__Arc_t {
   char md5[32 + 2];
 } PT_SaveCache__Arc_t;
 
-/* Append src to an .arc header block of capacity size, clipping what does not
-   fit: the values come from a cache entry, so shortening beats dropping it.
-   HTS_FALSE when it had to clip. */
-static hts_boolean arc_headers_cat(char *headers, size_t size,
-                                   const char *src) {
-  const size_t used = strlen(headers);
-  const size_t left = used < size - 1 ? size - used - 1 : 0;
-
-  if (left != 0) {
-    strlncatbuff(headers, src, size, left);
-  }
-  return strlen(src) <= left ? HTS_TRUE : HTS_FALSE;
-}
-
 static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element) {
   PT_SaveCache__Arc_t *st = (PT_SaveCache__Arc_t *) arg;
   FILE *const fp = st->fp;
   struct tm *tm = convert_time_rfc822(&st->buff, element->lastmodified);
   struct tm unknown_date;
-  /* the two strcatbuff calls closing the block rely on these 4 bytes */
-  const size_t room = sizeof(st->headers) - 4;
-  hts_boolean fit;
   int size_headers;
 
   /* a cached entry with no parseable Last-Modified must not take the writer
@@ -2402,42 +2388,34 @@ static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element)
     tm = &unknown_date;
   }
 
-  fit = slprintfbuff(st->headers, room,
-                     "HTTP/1.0 %d %s"
-                     "\r\n"
-                     "X-Server: ProxyTrack " PROXYTRACK_VERSION "\r\n"
-                     "Content-type: %s%s%s%s"
-                     "\r\n"
-                     "Last-modified: %s"
-                     "\r\n"
-                     "Content-length: %d"
-                     "\r\n",
-                     element->statuscode, element->msg,
-                     /**/ hts_effective_mime(element->contenttype),
-                     (element->charset[0] ? "; charset=\"" : ""),
-                     (element->charset[0] ? element->charset : ""),
-                     (element->charset[0] ? "\"" : ""),
-                     /**/ element->lastmodified, (int) element->size);
+  sprintf(st->headers,
+          "HTTP/1.0 %d %s"
+          "\r\n"
+          "X-Server: ProxyTrack " PROXYTRACK_VERSION "\r\n"
+          "Content-type: %s%s%s%s"
+          "\r\n"
+          "Last-modified: %s"
+          "\r\n"
+          "Content-length: %d"
+          "\r\n",
+          element->statuscode, element->msg,
+          /**/ hts_effective_mime(element->contenttype),
+          (element->charset[0] ? "; charset=\"" : ""),
+          (element->charset[0] ? element->charset : ""),
+          (element->charset[0] ? "\"" : ""), /**/ element->lastmodified,
+          (int) element->size);
   if (element->location != NULL && element->location[0] != '\0') {
-    if (!arc_headers_cat(st->headers, room, "Location: ") ||
-        !arc_headers_cat(st->headers, room, element->location) ||
-        !arc_headers_cat(st->headers, room, "\r\n")) {
-      fit = HTS_FALSE;
+    sprintf(st->headers + strlen(st->headers), "Location: %s" "\r\n",
+            element->location);
+  }
+  if (element->headers != NULL) {
+    if (strlen(element->headers) <
+        sizeof(st->headers) - strlen(element->headers) - 1) {
+      strcat(st->headers, element->headers);
     }
   }
-  if (element->headers != NULL &&
-      !arc_headers_cat(st->headers, room, element->headers)) {
-    fit = HTS_FALSE;
-  }
-  /* a clip landing mid-line must still end it, or the body reads as a header */
-  if (hts_lastchar(st->headers) != '\n') {
-    strcatbuff(st->headers, "\r\n");
-  }
-  strcatbuff(st->headers, "\r\n");
+  strcat(st->headers, "\r\n");
   size_headers = (int) strlen(st->headers);
-  if (!fit) {
-    fprintf(stderr, "Headers of %s clipped to %d bytes" LF, url, size_headers);
-  }
 
   /* doc == <nl><URL-record><nl><network_doc> */
 

src/punycode.c

@@ -425,11 +425,10 @@ int main(int argc, char **argv) {
       fail(io_error);
     if (feof(stdin))
       fail(invalid_input);
-    /* a leading NUL byte leaves strlen() at 0, so trim before subtracting */
-    input_length = strlen(input);
-    if (input_length == 0 || input[input_length - 1] != '\n')
+    input_length = strlen(input) - 1;
+    if (input[input_length] != '\n')
       fail(too_big);
-    input[--input_length] = 0;
+    input[input_length] = 0;
 
     for(p = input; *p != 0; ++p) {
       pp = strchr(print_ascii, *p);