tests: assert the strjoker budget fired, not just that work stayed bounded (#553)

The filterbounds self-test added in #551 asserted `steps < 10*maxsteps`, which
catches a budget whose enforcement is dropped (the counter still climbs to
~1.26e9). But the counter and the enforcement are the same statement, so
deleting the budget line outright leaves steps at 0: `0 < 10*maxsteps` holds and
the probe passes. Only the trailing strjokerfind call then catches it, by
hanging into a harness timeout, which is exactly what #551 set out to remove.

Add the floor `steps > maxsteps`: the budget must push the counter past the cap.
Deleting the budget now aborts the test in milliseconds instead of timing out.
Verified both regressions (line deleted; enforcement dropped) trip an assertion.

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.github/workflows/ci.yml

@@ -55,7 +55,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -99,7 +101,8 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          make check
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -132,13 +135,52 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
+      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
+        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
+        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
+        run: |
+          set -euo pipefail
+          sudo ifconfig lo0 alias 127.0.0.2 up
+          sudo ifconfig lo0 alias 127.0.0.3 up
+
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
+  # browser path (open -W) and nothing else exercises htsserver. Install into a
+  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
+  webhttrack-macos:
+    name: webhttrack smoke (macOS arm64)
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          brew install autoconf automake libtool autoconf-archive
+
+      - name: Build and install into a temp prefix
+        run: |
+          set -euo pipefail
+          ssl="$(brew --prefix openssl@3)"
+          autoreconf -fi
+          ./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
+            --prefix="$RUNNER_TEMP/inst"
+          make -j"$(sysctl -n hw.ncpu)"
+          make install
+
+      - name: Smoke-test webhttrack
+        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
+
   # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
   # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
   # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
@@ -171,7 +213,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -226,7 +270,9 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -272,7 +318,8 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          make check TESTS="$tests"
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs" TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
@@ -347,7 +394,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()

AGENTS.md

@@ -6,8 +6,19 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make && make check`. Or run
-  `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
+  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
+  automake's parallel harness and each crawl test binds its own ephemeral-port
+  server, so `-j` never contends and a multi-minute serial run drops to
+  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
+  automatically; only a test slower than the current longest raises the floor.
+  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
+  spend much of their wall time asleep (server trickles, httrack self-pacing),
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
+  platform, macOS included: the test server raises its listen backlog
+  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
+  `-c16` bigcrawl the way Python's default backlog of 5 did.
+  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

README

@@ -40,7 +40,6 @@ These are the principals limits of HTTrack for that moment. Note that we did not
 that would have solved them.
 
 - Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
-- Some java classes may not find some files on them (class included)
 - Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*
 
 

configure.ac

@@ -64,16 +64,6 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
 # Export version info
 AC_SUBST(VERSION_INFO)
 
-# Versioned plugin name for dlopen() in hts_create_opt(); soname major is
-# libtool's current - age, so this tracks VERSION_INFO bumps automatically.
-HTS_SONAME_MAJOR=$((${VERSION_INFO%%:*} - ${VERSION_INFO##*:}))
-case "$host_os" in
-darwin*) HTS_LIBHTSJAVA_NAME="libhtsjava.$HTS_SONAME_MAJOR.dylib" ;;
-*) HTS_LIBHTSJAVA_NAME="libhtsjava.so.$HTS_SONAME_MAJOR" ;;
-esac
-AC_DEFINE_UNQUOTED([HTS_LIBHTSJAVA_NAME], ["$HTS_LIBHTSJAVA_NAME"],
-  [Versioned libhtsjava runtime name, derived from VERSION_INFO])
-
 ### Default CFLAGS
 DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
 -Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
@@ -119,8 +109,8 @@ case "$CFLAGS" in
 esac
 
 # Force libc back into DT_NEEDED for libraries that reach it only through
-# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
-# linker; Apple ld rejects these flags and links libSystem unconditionally.
+# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
+# ld rejects these flags and links libSystem unconditionally.
 AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
 	[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
 AC_SUBST([LIBC_FORCE_LINK])

debian/copyright

@@ -7,9 +7,8 @@ Files: *
 Copyright: 1998-2026 Xavier Roche and other contributors
 License: GPL-3+
 Comment:
- The engine includes contributions from Yann Philippot (src/htsjava.c,
- src/htsjava.h). htsbasenet.h links against the system OpenSSL library
- (originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
+ htsbasenet.h links against the system OpenSSL library (originally by Eric
+ Young); no OpenSSL/SSLeay code is bundled here.
 
 Files: src/minizip/*
 Copyright: 1998-2010 Gilles Vollant

debian/libhttrack-dev.files

@@ -1,5 +1,4 @@
 usr/include/httrack
 usr/lib/*/libhttrack.{so}
-usr/lib/*/libhtsjava.{so}
 usr/share/httrack/libtest
 usr/lib/*/httrack/libtest

debian/libhttrack3.files

@@ -1,3 +1,2 @@
 usr/lib/*/libhttrack.so.3*
-usr/lib/*/libhtsjava.so.3*
 usr/share/httrack/templates

html/faq.html

@@ -609,10 +609,9 @@ most mirrors do work. We still are working to improve the mirror quality of HTTr
 <br>
 
 </a><a NAME="Q2b">Q: <strong>Some Java applets do not work properly!</strong><br>
-A: <em>Java applets may not work in some cases, for example if HTTrack failed to detect all included classes
-or files called within the class file. Sometimes, Java applets need to be online, because remote files are
-directly caught. Finally, the site structure can be incompatible with the class (always try to keep the original site structure
-when you want to get Java classes)<br>
+A: <em>Java applets may not work in some cases, because HTTrack does not parse compiled class files to find
+the resources they load. Java applets often need to be online anyway, since remote files are fetched directly
+at runtime.<br>
 If there is no way to make some classes work properly, you can exclude them with the filters. 
 They will be available, but only online.
 </em>

html/fcguide.html

@@ -243,7 +243,7 @@ Build options:
 Spider options:
   bN accept cookies in cookies.txt (0=do not accept,* 1=accept) (--cookies[=N])
   u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (--check-type[=N])
-  j *parse Java Classes (j0 don't parse) (--parse-java[=N])
+  j *parse scripts (j0 don't parse) (--parse-java[=N])
   sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N])
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (--http-10)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!) (--tolerant)
@@ -417,7 +417,7 @@ site. Specifically, the defauls are:
   X *purge old files after update (X0 keep delete)
   bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
   u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
-  j *parse Java Classes (j0 don't parse)
+  j *parse scripts (j0 don't parse)
   sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
   C  create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)
   f *log file mode
@@ -497,9 +497,9 @@ about your browser retaining them.
 <p align=justify> This causes different document types to be analyzed
 differently.
 
-<pre><b><i>  j *parse Java Classes (j0 don't parse) </i></b></pre>
+<pre><b><i>  j *parse scripts (j0 don't parse) </i></b></pre>
 
-<p align=justify> This causes Java class files to be parsed looking for
+<p align=justify> This causes scripts to be parsed looking for
 URLs.
 
 <pre><b><i>  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) </i></b></pre>
@@ -1178,7 +1178,7 @@ accepted and sent back in order to allow access.
 Spider options:
  bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
  u   check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
- j   *parse Java Classes (j0 don't parse)
+ j   *parse scripts (j0 don't parse)
  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!)
@@ -2451,12 +2451,10 @@ work.  We still are working to improve the mirror quality of HTTrack.
 
 <p align=justify><b>Q: Some Java applets do not work properly! </b>
 
-<p align=justify> A: Java applets may not work in some cases, for
-example if HTTrack failed to detect all included classes or files called
-within the class file.  Sometimes, Java applets need to be online,
-because remote files are directly caught.  Finally, the site structure
-can be incompatible with the class (always try to keep the original site
-structure when you want to get Java classes)
+<p align=justify> A: Java applets may not work in some cases, because
+HTTrack does not parse compiled class files to find the resources they
+load.  Java applets often need to be online anyway, since remote files
+are fetched directly at runtime.
 
 <p align=justify> If there is no way to make some classes work properly,
 you can exclude them with the filters.  They will be available, but only

html/httrack.man.html

@@ -764,9 +764,9 @@ check, * u1 check but /, u2 check always)
 <td width="78%">
 
 
-<p>*parse Java Classes (j0 don t parse, bitmask: |1 parse
-default, |2 don t parse .class |4 don t parse .js |8 don t
-be aggressive) (--parse-java[=N])</p></td></tr>
+<p>*parse scripts (j0 don t parse, bitmask: |1 parse
+default, |4 don t parse .js |8 don t be aggressive)
+(--parse-java[=N])</p></td></tr>
 <tr valign="top" align="left">
 <td width="11%"></td>
 <td width="6%">
@@ -889,8 +889,8 @@ only) (--protocol[=N])</p></td></tr>
 <td width="78%">
 
 
-<p>disable a specific external mime module (-%w htsswf -%w
-htsjava) (--disable-module &lt;param&gt;)</p></td></tr>
+<p>disable a specific external mime module (-%w
+httrack-plugin) (--disable-module &lt;param&gt;)</p></td></tr>
 </table>
 
 <p style="margin-left:11%; margin-top: 1em"><b>Browser
@@ -2175,10 +2175,6 @@ Several scripts generating complex filenames may not find
 them (ex:
 img.src=&rsquo;image&rsquo;+a+Mobj.dst+&rsquo;.gif&rsquo;)</small></p>
 
-<p style="margin-left:11%; margin-top: 1em"><small>- Some
-java classes may not find some files on them (class
-included)</small></p>
-
 <p style="margin-left:11%; margin-top: 1em"><small>-
 Cgi-bin links may not work properly in some cases
 (parameters needed). To avoid them: use filters like

html/options.html

@@ -288,8 +288,8 @@ This option define whether the engine has to abandon a host if a timeout/"too sl
 &P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
 This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
 
-j *parse Java Classes (j0 don't parse)
-This option define whether the engine has to parse java files or not to catch included files
+j *parse scripts (j0 don't parse)
+This option defines whether the engine has to parse scripts or not to catch included files
 
 I *make an index (I0 don't make)
 This option define whether the engine has to generate an index.html on the top directory

html/plug.html

@@ -103,220 +103,219 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
 
 <br>
 
-You can write external functions to be plugged in the httrack library very easily. 
-We'll see there some examples.
+You can write external functions to be plugged in the httrack library very easily. 
+We'll see there some examples.
 
 <br><br>
-
-The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
-The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
-<ul>
-<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
-<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
-<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
-</ul>
-
-<br>
-References: 
-<ul>
-<li>the <tt>httrack-library.h</tt> prototype file
-<br />
+
+The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
+The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
+<ul>
+<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
+<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
+<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
+</ul>
+
+<br>
+References: 
+<ul>
+<li>the <tt>httrack-library.h</tt> prototype file
+<br />
 Note: the <i>Initialization</i>, <i>Main functions</i>, <i>Options handling</i> and <i>Wrapper functions</i> sections are generally the only ones to be considered.
 </li>
-<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
-<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
-<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
-<li>the <tt>htsjava.c</tt> source file (the java class plugin ; overrides 'detect' and 'parse')</li>
-<li>the example given at the end of this document</li>
-</ul>
-
-<br />
-Below the list of functions to be defined in the module (plugin).<br />
-<br />
-
-<table width="100%">
-<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
-<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
-The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
-
-<br />
-Wrappers can be plugged inside hts_plug() using:<br />
-<tt>
-CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
-</tt>
-<br />
-
-<br />
-Example:
-<br />
-<tt>
-CHAIN_FUNCTION(opt, check_html, process, userdef);
-</tt>
-<br />
-
-</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
-
-<!-- -->
-
-<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
-The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
-
-</table>
-
-
-<br />
-Note that all callbacks (except init and uninit) take as first two argument:
-<ul>
-<li>the t_hts_callbackarg structure<br />
-this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
-</li>
-<li>the httrackp structure<br />
-this structure, holding all current httrack options and mirror state, can be read or mofidied
-</li>
-</ul>
-
-<br />
-Below the list of callbacks, and associated external wrappers.
-
-<table width="100%">
-<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
-
-<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-
-
-</table>
+<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
+<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
+<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
+<li>the example given at the end of this document</li>
+</ul>
+
+<br />
+Below the list of functions to be defined in the module (plugin).<br />
+<br />
+
+<table width="100%">
+<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
+<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
+The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
+
+<br />
+Wrappers can be plugged inside hts_plug() using:<br />
+<tt>
+CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
+</tt>
+<br />
+
+<br />
+Example:
+<br />
+<tt>
+CHAIN_FUNCTION(opt, check_html, process, userdef);
+</tt>
+<br />
+
+</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
+
+<!-- -->
+
+<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
+The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
+
+</table>
+
+
+<br />
+Note that all callbacks (except init and uninit) take as first two argument:
+<ul>
+<li>the t_hts_callbackarg structure<br />
+this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
+</li>
+<li>the httrackp structure<br />
+this structure, holding all current httrack options and mirror state, can be read or mofidied
+</li>
+</ul>
+
+<br />
+Below the list of callbacks, and associated external wrappers.
+
+<table width="100%">
+<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
+
+<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+
+
+</table>
 
 <br><br>
-Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
-
+Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
+
+
+<br />
+An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
+<table width="100%" border="2">
+<tr><td>
+<pre>
+/* system includes */
+#include &lt;stdio.h&gt;
+#include &lt;stdlib.h&gt;
+#include &lt;string.h&gt;
+
+/* standard httrack module includes */
+#include "httrack-library.h"
+#include "htsopt.h"
+#include "htsdefines.h"
+
+/* local function called as "check_html" callback */
+static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
+                        /*other parameters are callback-specific*/
+                        char* html, int len, const char* url_address, const char* url_file) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
+    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
+                                                html, len, url_address, url_file)) {
+        return 0;  /* abort */
+      }
+  }
+
+  printf("file %s%s content: %s\n", url_address, url_file, html);
+  return 1;  /* success */
+}
+
+/* local function called as "end" callback */
+static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* processing */
+  fprintf(stderr, "That's all, folks!\n");
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
+    /* status is ok on our side, return other callabck's status */
+    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
+  }
+
+  return 1;  /* success */
+}
+
+/*
+module entry point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
+  /* optional argument passed in the commandline we won't be using here */
+  const char *arg = strchr(argv, ',');
+  if (arg != NULL)
+    arg++;
+
+  /* plug callback functions */
+  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
+  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
+
+  return 1;  /* success */
+}
+
+/*
+module exit point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
+  fprintf(stder, "Module unplugged");
+
+  return 1;  /* success */
+}
+</pre>
+</td></tr></table>
+
+<br />
+Compile this file ; for example:
+<br />
+<tt>
+gcc -O -g3 -shared -o mylibrary.so myexample.c
+</tt>
+<br />
+and plug the module using the commandline ; for example:
+<br />
+<tt>
+httrack --wrapper mylibrary http://www.example.com
+</tt>
+<br />
+or, if some parameters are desired:
+<br />
+<tt>
+httrack --wrapper mylibrary,myparameter-string http://www.example.com
+</tt>
+<br />
+(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
+<br />
+
 
-<br />
-An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
-<table width="100%" border="2">
-<tr><td>
-<pre>
-/* system includes */
-#include &lt;stdio.h&gt;
-#include &lt;stdlib.h&gt;
-#include &lt;string.h&gt;
-
-/* standard httrack module includes */
-#include "httrack-library.h"
-#include "htsopt.h"
-#include "htsdefines.h"
-
-/* local function called as "check_html" callback */
-static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
-                        /*other parameters are callback-specific*/
-                        char* html, int len, const char* url_address, const char* url_file) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
-    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
-                                                html, len, url_address, url_file)) {
-        return 0;  /* abort */
-      }
-  }
-
-  printf("file %s%s content: %s\n", url_address, url_file, html);
-  return 1;  /* success */
-}
-
-/* local function called as "end" callback */
-static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* processing */
-  fprintf(stderr, "That's all, folks!\n");
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
-    /* status is ok on our side, return other callabck's status */
-    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
-  }
-
-  return 1;  /* success */
-}
-
-/*
-module entry point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
-  /* optional argument passed in the commandline we won't be using here */
-  const char *arg = strchr(argv, ',');
-  if (arg != NULL)
-    arg++;
-
-  /* plug callback functions */
-  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
-  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
-
-  return 1;  /* success */
-}
-
-/*
-module exit point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
-  fprintf(stder, "Module unplugged");
-
-  return 1;  /* success */
-}
-</pre>
-</td></tr></table>
-
-<br />
-Compile this file ; for example:
-<br />
-<tt>
-gcc -O -g3 -shared -o mylibrary.so myexample.c
-</tt>
-<br />
-and plug the module using the commandline ; for example:
-<br />
-<tt>
-httrack --wrapper mylibrary http://www.example.com
-</tt>
-<br />
-or, if some parameters are desired:
-<br />
-<tt>
-httrack --wrapper mylibrary,myparameter-string http://www.example.com
-</tt>
-<br />
-(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
-<br />
-
-
 <br><br>
 
 <!-- ==================== Start epilogue ==================== -->

html/step9_opt6.html

@@ -118,8 +118,8 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
   <br>Avoid "never", because the local mirror could be bogus
   </small><br><br>
 <!-- -->
-  <li>Parse java files</li>
-  <br><small>Must the engine parse .java files (java classes) to seek included filenames?
+  <li>Parse scripts</li>
+  <br><small>Must the engine parse scripts to seek included filenames?
   <br>It is checked by default
   </small><br><br>
 <!-- -->

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "07 July 2026" "httrack website copier"
+.TH httrack 1 "12 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -224,7 +224,7 @@ explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
-*parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
+*parse scripts (j0 don't parse, bitmask: |1 parse default, |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
 .IP \-sN
 follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules)) (\-\-robots[=N])
 .IP \-%h
@@ -248,7 +248,7 @@ can also be used to force a specific file type: \-\-assume foo.cgi=text/html
 .IP \-@iN
 internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only) (\-\-protocol[=N])
 .IP \-%w
-disable a specific external mime module (\-%w htsswf \-%w htsjava) (\-\-disable\-module <param>)
+disable a specific external mime module (\-%w httrack\-plugin) (\-\-disable\-module <param>)
 .SS Browser ID:
 .IP \-F
 user\-agent field sent in HTTP headers (\-F "user\-agent name") (\-\-user\-agent <param>)
@@ -514,8 +514,6 @@ that would have solved them.
 .SM
 \- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
 .SM
-\- Some java classes may not find some files on them (class included)
-.SM
 \- Cgi\-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like \-*cgi\-bin*
 .SH BUGS
 Please reports bugs to

src/Makefile.am

@@ -44,7 +44,7 @@ httrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 proxytrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 
-lib_LTLIBRARIES = libhttrack.la libhtsjava.la
+lib_LTLIBRARIES = libhttrack.la
 
 htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
 proxytrack_SOURCES = proxy/main.c \
@@ -82,14 +82,6 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $
 libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
 libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
 
-libhtsjava_la_SOURCES = htsjava.c htsjava.h
-libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
-# This thin JNI wrapper reaches libc only through libhttrack, so the direct
-# libc edge is dropped from DT_NEEDED (library-not-linked-against-libc). Force
-# libc back as a dependency; configure gates the flag since only a GNU-style
-# linker accepts it (LIBC_FORCE_LINK is empty on e.g. macOS).
-libhtsjava_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO) $(LIBC_FORCE_LINK)
-
 EXTRA_DIST = httrack.h webhttrack \
 		coucal/murmurhash3.h.diff \
 		coucal/murmurhash3.h.orig \
@@ -119,7 +111,6 @@ EXTRA_DIST = httrack.h webhttrack \
 		coucal/README.md \
 		coucal/sample.c \
 		coucal/tests.c \
-		htsjava.vcproj \
 		httrack.dsp httrack.dsw httrack.vcproj \
 		libhttrack.dsp libhttrack.dsw libhttrack.vcproj \
 		webhttrack.dsp webhttrack.dsw webhttrack.vcproj

src/htsback.c

@@ -68,6 +68,15 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
+/* Which hard quota, if any, is currently aborting the mirror. */
+typedef enum {
+  HTS_MIRROR_LIMIT_NONE = 0,
+  HTS_MIRROR_LIMIT_SIZE,
+  HTS_MIRROR_LIMIT_TIME,
+} hts_mirror_limit;
+
+static hts_mirror_limit back_mirror_limit(httrackp *opt);
+
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -524,22 +533,22 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
+static int create_back_tmpfile(httrackp *opt, lien_back *const back,
+                               const char *ext) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
-             back->url_sav);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
+             back->url_sav, ext);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
+      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
                     back->tmpfile);
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
-             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
-             opt->state.tmpnameid++);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
+             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -547,6 +556,32 @@ static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
   return 0;
 }
 
+/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
+   existing file moved the good copy to back->tmpfile before truncating url_sav.
+   commit keeps the new file and drops the backup; else restore it so an aborted
+   transfer leaves the previous copy intact. Skips the zlib .z temp. */
+static void back_finalize_backup(httrackp *opt, lien_back *const back,
+                                 const hts_boolean commit) {
+  if (back->tmpfile == NULL || back->r.compressed)
+    return;
+  if (commit) {
+    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
+  } else {
+    if (back->r.out != NULL) {
+      fclose(back->r.out);
+      back->r.out = NULL;
+    }
+    (void) UNLINK(back->url_sav); /* drop the failed partial */
+    /* On rename failure keep the backup: it still holds the good copy, so
+       losing it would be worse than an orphaned temp. */
+    if (RENAME(back->tmpfile, back->url_sav) != 0)
+      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                    "could not restore %s; previous copy kept as %s",
+                    back->url_sav, back->tmpfile);
+  }
+  back->tmpfile = NULL;
+}
+
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -582,6 +617,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
+      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -600,7 +636,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p]) == 0) {
+              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -673,6 +709,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
+        /* Body fully received: keep the freshly written url_sav, drop the
+           backup of the previous copy. */
+        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -901,6 +940,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
+  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
+     previous copy from the backup. */
+  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -2384,9 +2426,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
-     starves (#481). FTP slots stay, their thread owns the socket. */
+  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
+     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
   if (!back_checkmirror(opt)) {
+    const hts_mirror_limit limit = back_mirror_limit(opt);
+    const char *const reason =
+        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
+    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
+                                    ? "Mirror Size Limit"
+                                    : "Mirror Time Out";
     int aborted = 0;
     unsigned int i;
 
@@ -2400,15 +2448,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
           back_delayed_discard(opt, &back[i]);
         back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        strcpybuff(back[i].r.msg, slotmsg);
         back[i].status = STATUS_READY;
         back_set_finished(sback, i);
         aborted++;
       }
     }
     if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING,
-                    "time limit reached, %d transfer(s) aborted", aborted);
+      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
+                    reason, aborted);
     return;
   }
 
@@ -2898,7 +2946,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i]) == 0) {
+                              if (create_back_tmpfile(opt, &back[i], "z") ==
+                                  0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -2911,6 +2960,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
+                              /* Re-fetch over an existing file (#77 follow-up):
+                                 move the good copy aside before truncating it
+                                 so an aborted transfer can restore it. url_sav
+                                 is still written normally (file list intact).
+                               */
+                              back[i].tmpfile = NULL;
+                              if (fexist_utf8(back[i].url_sav)) {
+                                if (create_back_tmpfile(opt, &back[i], "bak") !=
+                                        0 ||
+                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
+                                        0) {
+                                  back[i].tmpfile = NULL;
+                                }
+                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3449,6 +3512,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
+                  // The server really sent 304 here; the *-hacks below force
+                  // NOT_MODIFIED only after confirming the file is complete.
+                  const hts_boolean server_sent_304 =
+                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
+
                   /*
                      Solve "false" 416 problems
                    */
@@ -3477,22 +3545,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // 'do not erase already downloaded file'
-                  // on an updated file
-                  // with an error : consider a 304 error
-                  if (!opt->delete_old) {
-                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
-                        && !back[i].testmode) {
-                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
-                        hts_log_print(opt, LOG_DEBUG,
-                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
-                                      back[i].r.statuscode, back[i].r.msg,
-                                      back[i].url_adr, back[i].url_fil);
-                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                        deletehttp(&back[i].r);
-                        back[i].r.soc = INVALID_SOCKET;
-                      }
-                    }
+                  // On update, keep the good copy on error (mask as 304); skip
+                  // resume paths so a stale-partial 416 still re-fetches.
+                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
+                      back[i].is_update && !back[i].testmode &&
+                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
+                      fexist_utf8(back[i].url_sav)) {
+                    hts_log_print(opt, LOG_DEBUG,
+                                  "Error %d (%s) ignored on update, keeping "
+                                  "existing copy: %s%s",
+                                  back[i].r.statuscode, back[i].r.msg,
+                                  back[i].url_adr, back[i].url_fil);
+                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                    deletehttp(&back[i].r);
+                    back[i].r.soc = INVALID_SOCKET;
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
@@ -3545,7 +3611,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) {       // 'OK'
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) {  // not HTML
                           if (strnotempty(back[i].url_sav)) {   // target found
-                            int size = fsize_utf8(back[i].url_sav);     // target size
+                            off_t size = fsize_utf8(back[i].url_sav);
 
                             if (size >= 0) {
                               if (back[i].r.totalsize == size) {        // same size!
@@ -3640,6 +3706,21 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
+                  // Out-of-protocol 304 to a Range resume: the file is still
+                  // partial, so drop it and refetch instead of trusting it.
+                  if (server_sent_304 && back[i].range_req_size > 0) {
+                    url_savename_refname_remove(opt, back[i].url_adr,
+                                                back[i].url_fil);
+                    UNLINK(back[i].url_sav);
+                    deletehttp(&back[i].r);
+                    back[i].r.soc = INVALID_SOCKET;
+                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
+                    strcpybuff(back[i].r.msg,
+                               "Bogus 304 on resume, restarting");
+                    back[i].status = STATUS_READY;
+                    back_set_finished(sback, i);
+                  }
+
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -3758,7 +3839,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       const hts_boolean range_ok =
                           back[i].r.crange > 0 && resume >= 0 &&
                           resume <= (LLint) sz &&
-                          back[i].r.crange_end + 1 == back[i].r.crange &&
+                          back[i].r.crange_end == back[i].r.crange - 1 &&
                           (back[i].r.totalsize < 0 ||
                            back[i].r.totalsize ==
                                back[i].r.crange_end - resume + 1);
@@ -3827,11 +3908,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           if (fp) {
                             LLint alloc_mem = resume + 1;
 
-                            if (back[i].r.totalsize >= 0)
+                            // Bound the in-memory buffer to a 32-bit size (real
+                            // in-RAM resources are far smaller); a hostile
+                            // Content-Length that would overflow the add or the
+                            // (size_t) cast is dropped and refetched instead.
+                            if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
+                              url_savename_refname_remove(opt, back[i].url_adr,
+                                                          back[i].url_fil);
+                              UNLINK(back[i].url_sav);
+                              alloc_mem = -1;
+                            } else if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
-                            if (deleteaddr(&back[i].r)
-                                && (back[i].r.adr =
-                                    (char *) malloct((size_t) alloc_mem))) {
+                            if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
+                                (back[i].r.adr =
+                                     (char *) malloct((size_t) alloc_mem))) {
                               back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
                                 back[i].r.totalsize += resume; // -> full size
@@ -4113,38 +4203,46 @@ static int back_maxtime_grace(const int maxtime) {
   return maximum(5, minimum(30, maxtime / 10));
 }
 
-int back_checkmirror(httrackp * opt) {
-  // Check max size
-  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
-    if (!opt->state.stop) {     /* not yet stopped */
-      hts_log_print(opt, LOG_ERROR,
-                    "More than " LLintP
-                    " bytes have been transferred.. giving up",
-                    (LLint) opt->maxsite);
-      /* cancel mirror smoothly */
-      hts_request_stop(opt, 0);
-    }
-    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
-    /*return 0;
-     */
-  }
-  // Check max time
+/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
+   No floor (unlike the time grace): a size overrun should abort promptly. */
+static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
+
+/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
+   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
+   which undercounts redirect/error-heavy crawls (#520). */
+static hts_mirror_limit back_mirror_limit(httrackp *opt) {
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
+          back_maxsize_grace(opt->maxsite))
+    return HTS_MIRROR_LIMIT_SIZE;
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
 
-    if (elapsed >= opt->maxtime) {
-      if (!opt->state.stop) { /* not yet stopped */
-        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                      opt->maxtime);
-        /* cancel mirror smoothly */
-        hts_request_stop(opt, 0);
-      }
-      /* smooth stop starved past the grace period: stop waiting (#481) */
-      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-        return 0;
-    }
+    if (elapsed >= opt->maxtime &&
+        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+      return HTS_MIRROR_LIMIT_TIME;
   }
-  return 1;                     /* Ok, go on */
+  return HTS_MIRROR_LIMIT_NONE;
+}
+
+int back_checkmirror(httrackp *opt) {
+  /* request a smooth stop the first time each cap is reached */
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+      !opt->state.stop) {
+    hts_log_print(opt, LOG_ERROR,
+                  "More than " LLintP
+                  " bytes have been transferred.. giving up",
+                  (LLint) opt->maxsite);
+    hts_request_stop(opt, 0);
+  }
+  if (opt->maxtime > 0 && !opt->state.stop &&
+      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
+    hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                  opt->maxtime);
+    hts_request_stop(opt, 0);
+  }
+  /* hard stop once a cap overruns its grace (callers must stop waiting) */
+  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
 }
 
 // octets transférés + add

src/htsback.h

@@ -141,8 +141,8 @@ LLint back_transferred(LLint add, struct_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
-   the -E deadline overran its grace period (callers must stop waiting). */
+/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
+   or -E deadline overruns its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htscore.c

@@ -1904,8 +1904,11 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
-          if (r.adr != NULL || r.size == 0) {
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
+          // An empty body must not overwrite the file when the transfer failed
+          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
+          // copy to 0 (#77 follow-up).
+          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1926,7 +1929,6 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
-
         }
 
         /* Parsing of other media types (java, ram..) */
@@ -3873,7 +3875,6 @@ void voidf(void) {
   a = "|                      HTTrack Website Copier   |" CRLF;
   a = "|Code:         Windows Interface Xavier Roche   |" CRLF;
   a = "|                    HTS/HTTrack Xavier Roche   |" CRLF;
-  a = "|                .class Parser Yann Philippot   |" CRLF;
   a = "|                                               |" CRLF;
   a = "|Tested on:                 Windows95,98,NT,2K  |" CRLF;
   a = "|                           Linux PC            |" CRLF;

src/htscoremain.c

@@ -1452,7 +1452,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF
                     ("Option %w needs to be followed by a blank space, and a module name");
-                  printf("Example: -%%w htsswf\n");
+                  printf("Example: -%%w httrack-plugin\n");
                   htsmain_free();
                   return -1;
                 } else {

src/htsdefines.h

@@ -70,7 +70,7 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 #endif
 
 /* Marks a symbol an external wrapper module exports back to the engine.
-   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins (htsjava)
+   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins
    hide their own hts_plug()/hts_unplug() entry points. */
 #ifndef EXTERNAL_FUNCTION
 #ifdef _WIN32

src/htsfilters.c

@@ -121,12 +121,18 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
   return use1 ? jok1 : jok2;
 }
 
+/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
+   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
+#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
+#define STRJOKER_MAXSTEPS 2000000u
+
 /* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
    pair keeps star-heavy patterns polynomial instead of exponential (#501). */
 typedef struct strjoker_memo {
   const char *chaine0, *joker0; /* offsets are relative to these bases */
   size_t stride;                /* strlen(joker0) + 1 */
   unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
+  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
 } strjoker_memo;
 
 static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
@@ -141,6 +147,8 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   size_t bit = 0;
   const char *adr;
 
+  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
+    return NULL; /* work budget spent: fail the match safely */
   if (memo->failed) {
     bit = (size_t) (chaine - memo->chaine0) * memo->stride +
           (size_t) (joker - memo->joker0);
@@ -153,22 +161,24 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   return adr;
 }
 
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                                LLint *size, int *size_flag) {
-  strjoker_memo memo = {chaine, joker, 0, NULL};
+/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
+   sweep passes one counter, bounding the whole scan). */
+static const char *strjoker_bounded(const char *chaine, const char *joker,
+                                    LLint *size, int *size_flag,
+                                    size_t *nsteps) {
+  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
   unsigned char stackbits[2048];
   hts_boolean onheap = HTS_FALSE;
   const char *adr;
 
   if (chaine != NULL && joker != NULL) {
-    const size_t n1 = strlen(chaine) + 1;
+    const size_t l1 = strlen(chaine), l2 = strlen(joker);
 
-    memo.stride = strlen(joker) + 1;
-    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = (n1 * memo.stride + 7) / 8;
+    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
+      return NULL; /* beyond any real filter/URL: bound depth+work */
+    memo.stride = l2 + 1;
+    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
 
       if (bytes <= sizeof(stackbits)) {
         memset(stackbits, 0, bytes);
@@ -185,6 +195,16 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
   return adr;
 }
 
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
+  size_t nsteps = 0;
+
+  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
+}
+
 /* Test-only oracle: the same matcher without the failure memo. */
 const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
                             int *size_flag) {
@@ -193,6 +213,20 @@ const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
   return strjoker_rec(&memo, chaine, joker, size, size_flag);
 }
 
+/* Test-only: strjoker() reporting the work-budget steps spent and the cap, so a
+   self-test can prove the budget bounds a hostile pattern's work. */
+const char *strjoker_steps(const char *chaine, const char *joker,
+                           size_t *nsteps_out, size_t *maxsteps_out) {
+  size_t nsteps = 0;
+  const char *r = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps);
+
+  if (nsteps_out != NULL)
+    *nsteps_out = nsteps;
+  if (maxsteps_out != NULL)
+    *maxsteps_out = STRJOKER_MAXSTEPS;
+  return r;
+}
+
 static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
                                  const char *joker, LLint *size,
                                  int *size_flag) {
@@ -431,12 +465,18 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
+  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
+  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
+    return NULL;
   while(*chaine) {
-    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
+    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
+                                &nsteps))) { // ok trouvé
       return adr;
     }
+    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
+      return NULL;
     chaine++;
   }
   return NULL;

src/htsfilters.h

@@ -52,6 +52,10 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
    oracle for the memoized matcher. */
 const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
                             int *size_flag);
+/* strjoker() reporting the work-budget steps it spent and the cap; test-only,
+   lets a self-test assert the budget bounds a hostile pattern's work. */
+const char *strjoker_steps(const char *chaine, const char *joker,
+                           size_t *nsteps_out, size_t *maxsteps_out);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsglobal.h

@@ -56,6 +56,10 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
+// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
+#include <stdint.h>
+#include <inttypes.h>
+
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -111,29 +115,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef SIZEOF_LONG_LONG
-#if SIZEOF_LONG_LONG == 8
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-#ifndef HTS_LONGLONG
-#ifdef __sun
-#define HTS_LONGLONG 0
-#endif
-#ifdef __osf__
-#define HTS_LONGLONG 0
-#endif
-#ifdef __linux
-#define HTS_LONGLONG 1
-#endif
-#ifdef _WIN32
-#define HTS_LONGLONG 1
-#endif
-#endif
-#endif
-
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -326,59 +307,11 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef HTS_NO_64_BIT
-#define HTS_LONGLONG 0
-#else
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-/* Wide integer types, chosen per platform.
-   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
-           plain int where 64-bit is unavailable).
-   TStamp: timestamp/duration in the same width (a double in the no-64-bit
-           fallback).
-   LLintP: the printf conversion for an LLint. */
-#if HTS_LONGLONG
-#ifdef LLINT_FORMAT
-typedef LLINT_TYPE LLint;
-
-typedef LLINT_TYPE TStamp;
-
-#define LLintP LLINT_FORMAT
-#else
-
-#ifdef _WIN32
-typedef __int64 LLint;
-
-typedef __int64 TStamp;
-
-#define LLintP "%I64d"
-#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
-       defined(__64BIT__))
-
-typedef long int LLint;
-
-typedef long int TStamp;
-
-#define LLintP "%ld"
-#else
-typedef long long int LLint;
-
-typedef long long int TStamp;
-
-#define LLintP "%lld"
-#endif
-
-#endif /* HTS_LONGLONG */
-
-#else
-typedef int LLint;
-
-#define LLintP "%d"
-typedef double TStamp;
-#endif
+/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
+typedef int64_t LLint;
+typedef int64_t TStamp;
+/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
+#define LLintP "%" PRId64
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain

src/htshelp.c

@@ -546,8 +546,8 @@ void help(const char *app, int more) {
   infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
-  infomsg
-    ("  j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
+  infomsg("  j *parse scripts (j0 don't parse, bitmask: |1 parse default, |4 "
+          "don't parse .js |8 don't be aggressive)");
   infomsg
     ("  sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
   infomsg
@@ -571,8 +571,7 @@ void help(const char *app, int more) {
     ("     can also be used to force a specific file type: --assume foo.cgi=text/html");
   infomsg
     (" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
-  infomsg
-    (" %w  disable a specific external mime module (-%w htsswf -%w htsjava)");
+  infomsg(" %w  disable a specific external mime module (-%w httrack-plugin)");
   infomsg("");
   infomsg("Browser ID:");
   infomsg

src/htsjava.c

@@ -1,517 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* ------------------------------------------------------------ */
-/* File: Java classes parser                                    */
-/* Author: Yann Philippot                                       */
-/* ------------------------------------------------------------ */
-
-/* Version: Oct/2000 */
-/* Fixed: problems with class structure (10/2000) */
-
-// htsjava.c - Parseur de classes java
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) )
-#include <sys/types.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-/* Standard httrack module includes */
-#include "httrack-library.h"
-#include "htsopt.h"
-#include "htsdefines.h"
-
-/* Module structures */
-#include "htsmodules.h"
-
-/* We link to libhttrack, we can use its functions */
-#include "httrack-library.h"
-
-/* This file */
-#include "htsjava.h"
-
-/* calloct/freet wrappers */
-#include "htssafe.h"
-
-static int reverse_endian(void) {
-  int endian = 1;
-
-  return (*((char *) &endian) == 1);
-}
-
-/* big/little endian swap */
-#define hts_swap16(A) ( (((A) & 0xFF)<<8) | (((A) & 0xFF00)>>8) )
-#define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) )
-
-/* Static definitions */
-static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT,
-                             int *);
-static unsigned short int readshort(FILE * fp);
-static int tris(httrackp * opt, char *);
-static char *printname(char[1024], char[1024]);
-
-// ** HTS_xx sinon pas pris par VC++
-#define HTS_CLASS  7
-#define HTS_FIELDREF  9
-#define HTS_METHODREF  10
-#define HTS_STRING  8
-#define HTS_INTEGER  3
-#define HTS_FLOAT  4
-#define HTS_LONG  5
-#define HTS_DOUBLE  6
-#define HTS_INTERFACE  11
-#define HTS_NAMEANDTYPE  12
-#define HTS_ASCIZ  1
-#define HTS_UNICODE  2
-
-#define JAVADEBUG 0
-
-static const char *libName = "htsjava";
-
-#ifdef _WIN32
-#define strcasecmp(a,b) stricmp(a,b)
-#define strncasecmp(a,b,n) strnicmp(a,b,n)
-#endif
-
-static int detect_mime(htsmoduleStruct * str) {
-  const char *savename = str->filename;
-
-  if (savename) {
-    int len = (int) strlen(savename);
-
-    if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) {
-      return 1;
-    }
-  }
-  return 0;
-}
-
-static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt,
-                           htsmoduleStruct * str) {
-  /* Call parent functions if multiple callbacks are chained. */
-  if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) {
-    if (CALLBACKARG_PREV_FUN(carg, detect)
-        (CALLBACKARG_PREV_CARG(carg), opt, str)) {
-      return 1;                 /* Found before us, let them have the priority */
-    }
-  }
-
-  /* Check MIME */
-  if (detect_mime(str)) {
-    str->wrapper_name = libName;        /* Our ID */
-    return 1;                   /* Known format, we take it */
-  }
-
-  return 0;                     /* Unknown format */
-}
-
-static off_t fsize(const char *s) {
-  STRUCT_STAT st;
-
-  if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
-    return st.st_size;
-  } else {
-    return -1;
-  }
-}
-
-static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
-                          htsmoduleStruct * str) {
-  /* The wrapper_name memebr has changed: not for us anymore */
-  if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) {
-    /* Call parent functions if multiple callbacks are chained. */
-    if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) {
-      return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg),
-                                                opt, str);
-    }
-    strcpy(str->err_msg,
-           "unexpected error: bad wrapper_name and no previous wrapper");
-    return 0;                   /* Unexpected error */
-  } else {
-    if (detect_mime(str)) {
-
-      /* (Legacy code) */
-      char catbuff[CATBUFF_SIZE];
-      FILE *fpout;
-      JAVA_HEADER header;
-      RESP_STRUCT *tab;
-      const char *file = str->filename;
-
-      str->relativeToHtmlLink = 1;
-
-#if JAVADEBUG
-      printf("fopen\n");
-#endif
-      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
-          NULL) {
-        sprintf(str->err_msg, "Unable to open file %s", file);
-        return 0;               // une erreur..
-      }
-#if JAVADEBUG
-      printf("fread\n");
-#endif
-      if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
-        fclose(fpout);
-        sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
-                (LLint) fsize(file));
-        return 0;
-      }
-#if JAVADEBUG
-      printf("header\n");
-#endif
-      // tester en tête
-      if (reverse_endian()) {
-        header.magic = hts_swap32(header.magic);
-        header.count = hts_swap16(header.count);
-      }
-      if (header.magic != 0xCAFEBABE) {
-        sprintf(str->err_msg, "non java file");
-        if (fpout) {
-          fclose(fpout);
-          fpout = NULL;
-        }
-        return 0;
-      }
-
-      /* A constant-pool entry is >= 1 byte on disk; reject a count exceeding
-         the file size (hostile .class ~68 MB alloc DoS). */
-      if (!hts_count_fits(header.count, (LLint) fsize(file))) {
-        fclose(fpout);
-        sprintf(str->err_msg,
-                "Invalid constant pool count %u (file len " LLintP ")",
-                (unsigned) header.count, (LLint) fsize(file));
-        return 0;
-      }
-      tab = (RESP_STRUCT *) calloct(header.count, sizeof(RESP_STRUCT));
-      if (!tab) {
-        sprintf(str->err_msg, "Unable to alloc %d bytes",
-                (int) sizeof(RESP_STRUCT));
-        if (fpout) {
-          fclose(fpout);
-          fpout = NULL;
-        }
-        return 0;               // erreur..
-      }
-#if JAVADEBUG
-      printf("calchead\n");
-#endif
-      {
-        int i;
-
-        for(i = 1; i < header.count; i++) {
-          int err = 0;          // ++    
-
-          tab[i] = readtable(str, fpout, tab[i], &err);
-          if (!err) {
-            if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE))
-              i++;              //2 element si double ou float
-          } else {              // ++ une erreur est survenue!
-            if (strnotempty(str->err_msg) == 0)
-              strcpy(str->err_msg, "Internal readtable error");
-            freet(tab);
-            if (fpout) {
-              fclose(fpout);
-              fpout = NULL;
-            }
-            return 0;
-          }
-        }
-
-      }
-
-#if JAVADEBUG
-      printf("addfiles\n");
-#endif
-      {
-        unsigned int Class;
-        unsigned int SClass;
-        int i;
-
-        Class = readshort(fpout);
-        SClass = readshort(fpout);
-
-        for(i = 1; i < header.count; i++) {
-
-          if (tab[i].type == HTS_CLASS) {
-
-            if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) {
-
-              if ((tab[i].index1 != SClass) && (tab[i].index1 != Class)
-                  && (tab[tab[i].index1].name[0] != '[')) {
-
-                if (!strstr(tab[tab[i].index1].name, "java/")) {
-                  char BIGSTK tempo[1024];
-
-                  tempo[0] = '\0';
-
-                  snprintf(tempo, sizeof(tempo), "%s.class", tab[tab[i].index1].name);
-#if JAVADEBUG
-                  printf("add %s\n", tempo);
-#endif
-                  if (tab[tab[i].index1].file_position >= 0)
-                    str->addLink(str, tempo);   /* tab[tab[i].index1].file_position */
-                }
-
-              }
-            } else {
-              i = header.count; // exit 
-            }
-          }
-
-        }
-      }
-
-#if JAVADEBUG
-      printf("end\n");
-#endif
-      freet(tab);
-      if (fpout) {
-        fclose(fpout);
-        fpout = NULL;
-      }
-      return 1;
-
-    } else {
-      strcpy(str->err_msg, "bad MIME type");
-    }
-  }
-  return 0;                     /* Error */
-}
-
-/*
-module entry point 
-*/
-EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
-EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
-  /* Plug callback functions */
-  CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL);
-  CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL);
-
-  return 1;                     /* success */
-}
-
-// error: !=0 si erreur fatale
-static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
-                             RESP_STRUCT trans, int *error) {
-  char rname[1024];
-  unsigned short int length;
-  int j;
-
-  *error = 0;                   // pas d'erreur
-  trans.file_position = -1;
-  trans.type = (int) (unsigned char) fgetc(fp);
-  switch (trans.type) {
-  case HTS_CLASS:
-    strcpy(trans.name, "Class");
-    trans.index1 = readshort(fp);
-    break;
-
-  case HTS_FIELDREF:
-    strcpy(trans.name, "Field Reference");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-
-  case HTS_METHODREF:
-    strcpy(trans.name, "Method Reference");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-
-  case HTS_INTERFACE:
-    strcpy(trans.name, "Interface Method Reference");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-  case HTS_NAMEANDTYPE:
-    strcpy(trans.name, "Name and Type");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-
-  case HTS_STRING:             // CONSTANT_String
-    strcpy(trans.name, "String");
-    trans.index1 = readshort(fp);
-    break;
-
-  case HTS_INTEGER:
-    strcpy(trans.name, "Integer");
-    for(j = 0; j < 4; j++)
-      fgetc(fp);
-    break;
-
-  case HTS_FLOAT:
-    strcpy(trans.name, "Float");
-    for(j = 0; j < 4; j++)
-      fgetc(fp);
-    break;
-
-  case HTS_LONG:
-    strcpy(trans.name, "Long");
-    for(j = 0; j < 8; j++)
-      fgetc(fp);
-    break;
-  case HTS_DOUBLE:
-    strcpy(trans.name, "Double");
-    for(j = 0; j < 8; j++)
-      fgetc(fp);
-    break;
-
-  case HTS_ASCIZ:
-  case HTS_UNICODE:
-
-    if (trans.type == HTS_ASCIZ)
-      strcpy(trans.name, "HTS_ASCIZ");
-    else
-      strcpy(trans.name, "HTS_UNICODE");
-
-    {
-      char BIGSTK buffer[1024];
-      char *p;
-
-      p = &buffer[0];
-
-      trans.file_position = ftell(fp);
-      length = readshort(fp);
-      if (length < HTS_URLMAXSIZE) {
-        while(length > 0) {
-          *p++ = fgetc(fp);
-
-          length--;
-        }
-        *p = '\0';
-
-        if (tris(str->opt, buffer) == 1)
-          str->addLink(str, buffer);    /* trans.file_position */
-        else if (tris(str->opt, buffer) == 2)
-          str->addLink(str, printname(rname, buffer));
-
-        strcpy(trans.name, buffer);
-      } else { // gros pb
-        while((length > 0) && (!feof(fp))) {
-          fgetc(fp);
-          length--;
-        }
-        if (!feof(fp)) {
-          trans.type = -1;
-        } else {
-          sprintf(str->err_msg, "Internal structure error (ASCII)");
-          *error = 1;
-        }
-        return (trans);
-      }
-    }
-    break;
-  default:
-    // on arrête tout 
-    sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
-    *error = 1;
-    return (trans);
-    break;
-  }
-  return (trans);
-}
-
-static unsigned short int readshort(FILE * fp) {
-  unsigned short int valint;
-
-  if (fread(&valint, sizeof(valint), 1, fp) == 1) {
-    if (reverse_endian())
-      return hts_swap16(valint);
-    else
-      return valint;
-  } else {
-    return 0;
-  }
-
-}
-
-static int tris(httrackp * opt, char *buffer) {
-  char catbuff[CATBUFF_SIZE];
-
-  //
-  // Java
-  if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/")))
-    return 2;
-  if (strstr(buffer, ".gif") || strstr(buffer, ".jpg")
-      || strstr(buffer, ".jpeg") || strstr(buffer, ".au"))
-    return 1;
-  // Ajouts R.X: test type
-  // Autres fichiers
-  {
-    char type[256];
-
-    if (get_httptype_sized(opt, type, sizeof(type), buffer,
-                           0)) // recognized type
-      return 1;
-    // ajout RX 05/2001
-    else if (is_dyntype(get_ext(catbuff, sizeof(catbuff), buffer)))      // asp,cgi...
-      return 1;
-  }
-  return 0;
-}
-
-static char *printname(char rname[1024], char name[1024]) {
-  char *p;
-  char *p1;
-  size_t j;
-
-  rname[0] = '\0';
-  //
-
-  p = &name[0];
-
-  if (*p != '[') {
-    return rname;  // ""
-  }
-  p += 2;
-  p1 = rname;
-  for(j = 0; name[j] != '\0'; j++, p++) {
-    if (*p == '/')
-      *p1 = '.';
-    if (*p == ';') {
-      *p1 = '\0';
-      strcat(rname, ".class");
-      return rname;
-    } else
-      *p1 = *p;
-    p1++;
-  }
-  p1 -= 3;
-  *p1 = '\0';
-  return rname;
-
-}

src/htsjava.h

@@ -1,70 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* ------------------------------------------------------------ */
-/* File: Java classes parser .h                                 */
-/* Author: Yann Philippot                                       */
-/* ------------------------------------------------------------ */
-
-#ifndef HTSJAVA_DEFH
-#define HTSJAVA_DEFH
-
-#include <stdint.h>
-
-#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
-#define HTS_DEF_FWSTRUCT_JAVA_HEADER
-typedef struct JAVA_HEADER JAVA_HEADER;
-#endif
-/* 10-byte on-disk .class header image, fread() directly: fields need exact
-   widths (LP64's 8-byte 'unsigned long' magic never matched 0xCAFEBABE). */
-struct JAVA_HEADER {
-  uint32_t magic;
-  uint16_t minor;
-  uint16_t major;
-  uint16_t count;
-};
-
-#ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT
-#define HTS_DEF_FWSTRUCT_RESP_STRUCT
-typedef struct RESP_STRUCT RESP_STRUCT;
-#endif
-struct RESP_STRUCT {
-  int file_position;
-  //
-  unsigned int index1;
-  unsigned int type;
-  char name[1024];
-};
-
-/* Library internal definictions */
-#ifdef HTS_INTERNAL_BYTECODE
-
-EXTERNAL_FUNCTION int hts_plug_java(httrackp * opt, const char *argv);
-
-#endif
-
-#endif

src/htsjava.vcproj

@@ -1,373 +0,0 @@
-<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
-	ProjectType="Visual C++"
-	Version="9,00"
-	Name="htsjava"
-	ProjectGUID="{2F353408-2C6C-4D31-9AC0-08BDC064DF64}"
-	RootNamespace="htsjava"
-	Keyword="Win32Proj"
-	TargetFrameworkVersion="131072"
-	>
-	<Platforms>
-		<Platform
-			Name="Win32"
-		/>
-		<Platform
-			Name="x64"
-		/>
-	</Platforms>
-	<ToolFiles>
-	</ToolFiles>
-	<Configurations>
-		<Configuration
-			Name="Debug|Win32"
-			OutputDirectory="C:\temp\htsjava\Debug"
-			IntermediateDirectory="C:\temp\htsjava\Debug"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				Optimization="0"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				MinimalRebuild="true"
-				BasicRuntimeChecks="3"
-				RuntimeLibrary="1"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="4"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="C:\temp\httrack\htsjava.dll"
-				LinkIncremental="2"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
-				SubSystem="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="1"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-			/>
-		</Configuration>
-		<Configuration
-			Name="Debug|x64"
-			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
-			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-				TargetEnvironment="3"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				Optimization="0"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				MinimalRebuild="true"
-				BasicRuntimeChecks="3"
-				RuntimeLibrary="1"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="3"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="C:\temp\httrack64\htsjava.dll"
-				LinkIncremental="2"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
-				SubSystem="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="17"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-			/>
-		</Configuration>
-		<Configuration
-			Name="Release|Win32"
-			OutputDirectory="C:\temp\htsjava\Release"
-			IntermediateDirectory="C:\temp\htsjava\Release"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				RuntimeLibrary="0"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="0"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="O:\HTTrack\httrack\htsjava.dll"
-				LinkIncremental="1"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="O:\HTTrack\httrack\htsjava.pdb"
-				SubSystem="2"
-				OptimizeReferences="2"
-				EnableCOMDATFolding="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="1"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
-			/>
-		</Configuration>
-		<Configuration
-			Name="Release|x64"
-			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
-			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-				TargetEnvironment="3"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				RuntimeLibrary="0"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="0"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="O:\HTTrack\httrack\x64\htsjava.dll"
-				LinkIncremental="1"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="O:\HTTrack\httrack\x64\htsjava.pdb"
-				SubSystem="2"
-				OptimizeReferences="2"
-				EnableCOMDATFolding="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="17"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
-			/>
-		</Configuration>
-	</Configurations>
-	<References>
-	</References>
-	<Files>
-		<Filter
-			Name="Source Files"
-			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
-			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
-			>
-			<File
-				RelativePath="htsjava.c"
-				>
-			</File>
-		</Filter>
-		<Filter
-			Name="Header Files"
-			Filter="h;hpp;hxx;hm;inl;inc;xsd"
-			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
-			>
-			<File
-				RelativePath="htsjava.h"
-				>
-			</File>
-		</Filter>
-		<Filter
-			Name="Resource Files"
-			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
-			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
-			>
-		</Filter>
-	</Files>
-	<Globals>
-	</Globals>
-</VisualStudioProject>

src/htslib.c

@@ -1059,10 +1059,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Self-test entry for append_cookie_header(): build the request Cookie line
-   into dst (always NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
-                                const char *path, char *dst, size_t dst_size) {
+/* Build the request Cookie line for domain/path into dst (always
+   NUL-terminated). Returns the number of cookies emitted. */
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
+                       char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -1586,7 +1586,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a = strchr(rcvd + p, '/');
         if (a != NULL) {
           a++;
-          if (sscanf(a, LLintP, &retour->crange) == 1) {
+          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
             retour->crange_start = 0;
             retour->crange_end = retour->crange - 1;
           } else {
@@ -1594,6 +1594,12 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
+      // A valid Content-Range has no negative field; reject hostile values so
+      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
+      if (retour->crange_start < 0 || retour->crange_end < 0 ||
+          retour->crange < 0) {
+        retour->crange_start = retour->crange_end = retour->crange = 0;
+      }
     }
   } else if ((p = strfield(rcvd, "Connection:")) != 0) {
     char *a = rcvd + p;
@@ -1702,6 +1708,9 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
 #if DEBUG_COOK
     printf("set-cookie detected\n");
 #endif
+    /* Over-long host overflows the fail-safe domain[] copy (abort); drop it. */
+    if (adr && strlen(jump_identification_const(adr)) >= sizeof(domain))
+      return;
     while(*a) {
       char *token_st, *token_end;
       char *value_st, *value_end;
@@ -1998,6 +2007,15 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
+      // In-memory content must fit a 32-bit index (allocs below add 1, reads
+      // use int offsets): reject a hostile Content-Length or endless stream.
+      const LLint inmem_want =
+          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
+      if (inmem_want >= INT32_MAX) {
+        r->statuscode = STATUSCODE_INVALID;
+        strcpybuff(r->msg, "In-memory content too large");
+        return READ_ERROR;
+      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);
@@ -2560,6 +2578,11 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     // recopier adrfil->adresse www..
     strncatbuff(adrfil->adr, p, ((int) (q - p)));
     // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!
+
+    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
+    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
+      return -1;
+
     // recopier chemin /pub/..
     if (q[0] != '/')            // page par défaut (/)
       strcatbuff(adrfil->fil, "/");
@@ -3038,12 +3061,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
-#ifdef HTS_LONGLONG
-#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#endif
+#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
+#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintPiB                                                             \
+  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3052,9 +3074,7 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  }
-#ifdef HTS_LONGLONG
-  else if (n < ToLLintGiB) {
+  } else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3071,13 +3091,6 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
-#else
-  else {
-    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
-            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
-    strcpybuff(strc->buff2, "MiB");
-  }
-#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;
@@ -5913,17 +5926,7 @@ int multipleStringMatch(const char *s, const char *match) {
 }
 
 HTSEXT_API httrackp *hts_create_opt(void) {
-#if ( defined(_WIN32) || defined(__ANDROID__) )
-  static const char *defaultModules[] = {
-    "htsswf", "htsjava", "httrack-plugin", NULL
-  };
-#else
-#ifndef HTS_LIBHTSJAVA_NAME
-#define HTS_LIBHTSJAVA_NAME "libhtsjava.so" /* non-autoconf fallback */
-#endif
-  static const char *defaultModules[] = {"libhtsswf.so.1", HTS_LIBHTSJAVA_NAME,
-                                         "httrack-plugin", NULL};
-#endif
+  static const char *defaultModules[] = {"httrack-plugin", NULL};
   httrackp *opt = malloc(sizeof(httrackp));
 
   /* default options */

src/htslib.h

@@ -184,10 +184,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
 /* Build the request "Cookie:" header line for stored cookies matching
-   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
-   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
-                                const char *path, char *dst, size_t dst_size);
+   domain/path into dst (NUL-terminated), wrapping the logic http_sendhead()
+   uses. Returns cookies emitted. */
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
+                       char *dst, size_t dst_size);
 
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);

src/htsparse.c

@@ -4311,6 +4311,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       *forbidden_url == 0 && IS_DELAYED_EXT(afs->save) && !opt->state.stop) {
     int loops;
     int continue_loop;
+    char BIGSTK
+        cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */
 
     hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", afs->af.adr,
                   afs->af.fil);
@@ -4319,6 +4321,12 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
     for(loops = 0, continue_loop = 1;
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
+      /* #15: snapshot the Cookie header THIS url would send, so only its own
+         self-redirect Set-Cookie trips the retry, not a concurrent slot's. */
+      cookie_before[0] = '\0';
+      if (opt->accept_cookie && opt->cookie != NULL)
+        http_cookie_header(opt->cookie, jump_identification_const(afs->af.adr),
+                           afs->af.fil, cookie_before, sizeof(cookie_before));
 
       /* Wait for an available slot */
       if (!hts_wait_available_socket(sback, opt, cache, ptr))
@@ -4516,6 +4524,17 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           /* Moved! */
           else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) {
             char BIGSTK mov_url[HTS_URLMAXSIZE * 2];
+            char BIGSTK cookie_after[16384];
+            hts_boolean cookies_changed;
+
+            /* #15: cookie-wall signal: this URL's own Cookie header changed
+               across its self-redirect (a Set-Cookie it just set). */
+            cookie_after[0] = '\0';
+            if (opt->accept_cookie && opt->cookie != NULL)
+              http_cookie_header(
+                  opt->cookie, jump_identification_const(afs->af.adr),
+                  afs->af.fil, cookie_after, sizeof(cookie_after));
+            cookies_changed = strcmp(cookie_before, cookie_after) != 0;
 
             mov_url[0] = '\0';
             strcpybuff(mov_url, back[b].r.location);    // copier URL
@@ -4585,11 +4604,24 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
                   url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, 
                                opt, sback, cache, hash, ptr, numero_passe,
                                &delayed_back);
+                } else if (cookies_changed) {
+                  // #15: cookie-wall self-redirect; evict the cached
+                  // fast-header so the re-issue refetches with the cookie.
+                  if (cache->cached_tests != NULL)
+                    coucal_remove(cache->cached_tests,
+                                  concat(OPT_GET_BUFF(opt),
+                                         OPT_GET_BUFF_SIZE(opt), afs->af.adr,
+                                         afs->af.fil));
+                  afs->save[0] = '\0';
+                  url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, opt,
+                               sback, cache, hash, ptr, numero_passe,
+                               &delayed_back);
+                  continue_loop = 1;
                 } else {
                   hts_log_print(opt, LOG_WARNING,
                                 "Unable to test %s%s (loop to same filename)",
                                 afs->af.adr, afs->af.fil);
-                }               // loop to same location
+                } // loop to same location
               }                 // ident_url_relatif()
             }                   // location
           }                     // redirect

src/htssafe.h

@@ -456,13 +456,6 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
   return b->buf;
 }
 
-/** True if 'count' records of >= 1 byte each fit in 'available' bytes; guards
-    an attacker-controlled count driving a large allocation. */
-static HTS_INLINE HTS_UNUSED hts_boolean hts_count_fits(size_t count,
-                                                        LLint available) {
-  return (available >= 0 && (LLint) count <= available) ? HTS_TRUE : HTS_FALSE;
-}
-
 /* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
    added bounds checking. freet() also NULLs the freed pointer and tolerates
    NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the

src/htsselftest.c

@@ -740,6 +740,44 @@ static int st_filterdual(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Length/work caps stop a hostile pattern stack-overflowing or hanging the
+   process (OSS-Fuzz 5060751291908096 / 5745936014573568). */
+static int st_filterbounds(httrackp *opt, int argc, char **argv) {
+  const size_t big = 100000; /* well past the length cap */
+  const size_t stars = 1023; /* pattern len 2047, under the length cap */
+  const size_t subjlen = 2048;
+  char *subj = malloct(big + 1);
+  char *pat = malloct(2 * stars + 2);
+  size_t steps = 0, maxsteps = 0, i;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  memset(subj, 'a', big);
+  subj[big] = '\0';
+  /* '*' matches anything, but an over-length subject trips the length cap */
+  assertf(strjoker(subj, "*", NULL, NULL) == NULL);
+  assertf(strjokerfind(subj, "*") == NULL);
+  /* Star-heavy dead-end at the length cap: unbounded it runs ~1.26e9 memo-steps
+     (~6s). */
+  for (i = 0; i < stars; i++) {
+    pat[2 * i] = '*';
+    pat[2 * i + 1] = 'a';
+  }
+  pat[2 * stars] = 'b'; /* never matches an all-'a' subject */
+  pat[2 * stars + 1] = '\0';
+  subj[subjlen] = '\0';
+  /* Budget must fire and hold: steps > cap (deleting the budget zeroes the
+     counter that is the enforcement), steps < 10*cap (unbudgeted ~1.26e9). */
+  assertf(strjoker_steps(subj, pat, &steps, &maxsteps) == NULL);
+  assertf(steps > maxsteps && steps < 10 * maxsteps);
+  assertf(strjokerfind(subj, pat) == NULL);
+  freet(pat);
+  freet(subj);
+  printf("filterbounds: OK\n");
+  return 0;
+}
+
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -1266,6 +1304,30 @@ static int st_identurl(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
+   to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
+static int st_identabs(httrackp *opt, int argc, char **argv) {
+  lien_adrfil af;
+  const size_t len =
+      sizeof(af.fil) - 1; /* 2047: max URL the top guard admits */
+  char *url = malloct(len + 1);
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  url[0] = '?';
+  memset(url + 1, 'a', len - 1);
+  url[len] = '\0';
+  assertf(ident_url_absolute(url, &af) == -1);
+  freet(url);
+  /* valid URLs still parse, so the guard is not over-rejecting */
+  assertf(ident_url_absolute("http://www.example.com/a/b/c.html?x=1", &af) ==
+          0);
+  assertf(ident_url_absolute("www.foo.com?bar=1", &af) == 0);
+  printf("identabs self-test OK\n");
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -1310,6 +1372,82 @@ static int st_headerlong(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* http_xfread1 must refuse an in-memory buffer whose size would exceed a 32-bit
+   index (hostile Content-Length or endless stream) rather than allocate it.
+   The guard returns before any socket read, so no real connection is needed. */
+static int st_xfread_limit(httrackp *opt, int argc, char **argv) {
+  htsblk r;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+  // Content-Length just over 2 GiB.
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = (LLint) INT32_MAX + 1;
+  printf("bylen: refused=%d adr=%s msg=%s\n",
+         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
+         r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+
+  // Unknown length, buffer already at the limit: the next read would exceed it.
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = -1;
+  r.size = (LLint) INT32_MAX;
+  printf("bygrow: refused=%d adr=%s msg=%s\n",
+         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
+         r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+
+  // Exactly at the 2 GiB index (size + bufl == INT32_MAX): must also be
+  // refused, since the reallocs below add 1 (a `> INT32_MAX` check would let
+  // this through and overflow the int realloc size).
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = -1;
+  r.size = (LLint) INT32_MAX - 8192;
+  http_xfread1(&r, 8192);
+  printf("boundary: msg=%s\n", r.msg);
+
+  // A legitimate small size must NOT be refused by the guard (the read then
+  // fails on the invalid socket, but the size-too-large msg must not be set).
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = 1000;
+  http_xfread1(&r, 8192);
+  printf("accept: msg=%s\n", r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+  return 0;
+}
+
+/* Parse a Content-Range header and print the sanitized triple. A hostile value
+   (negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
+static int st_crange(httrackp *opt, int argc, char **argv) {
+  int i;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "crange: needs at least one raw Content-Range line\n");
+    return 1;
+  }
+  for (i = 0; i < argc; i++) {
+    htsblk r;
+    char BIGSTK line[HTS_URLMAXSIZE * 2];
+
+    memset(&r, 0, sizeof(r));
+    strcpybuff(line, argv[i]);
+    treathead(NULL, "www.example.com", "/", &r, line);
+    printf("crange_start=" LLintP " crange_end=" LLintP " crange=" LLintP "\n",
+           (LLint) r.crange_start, (LLint) r.crange_end, (LLint) r.crange);
+  }
+  return 0;
+}
+
 /* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
 static size_t st_decode_body(const char *arg, char *buf, size_t size) {
   size_t n = 0;
@@ -1679,9 +1817,30 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     return 1;
   }
 
-  http_cookie_header_selftest(&cookie, dom, "/", hdr, sizeof(hdr));
+  http_cookie_header(&cookie, dom, "/", hdr, sizeof(hdr));
   if (strcmp(hdr, expected) != 0)
     err = 1;
+
+  /* A hostile over-long request host must not overflow domain[256] in
+     treathead's default-domain copy (that would abort the mirror). */
+  {
+    static t_cookie ck2;
+    htsblk r;
+    char host[600];
+
+    memset(&r, 0, sizeof(r));
+    memset(host, 'a', sizeof(host) - 1);
+    host[sizeof(host) - 1] = '\0';
+    ck2.max_len = (int) sizeof(ck2.data);
+    ck2.data[0] = '\0';
+    treathead(&ck2, host, "/", &r, "Set-Cookie: SID=1; path=/");
+    if (strnotempty(ck2.data)) // oversize-host cookie was not dropped
+      err = 1;
+    /* control: a normal host still yields a cookie through treathead */
+    treathead(&ck2, dom, "/", &r, "Set-Cookie: SID=1; path=/");
+    if (strstr(ck2.data, "SID") == NULL) // guard wrongly dropped a valid cookie
+      err = 1;
+  }
   if (strstr(hdr, "$Version") != NULL || strstr(hdr, "$Path") != NULL)
     err = 1;
   if (strstr(hdr, "junk") != NULL) // wrong-domain cookie leaked
@@ -2425,21 +2584,6 @@ static int st_ftpuser(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* hts_count_fits caps the .class constant-pool entry count to the file size,
-   rejecting the ~68 MB-per-file calloc DoS. */
-static int st_java(httrackp *opt, int argc, char **argv) {
-  (void) opt;
-  (void) argc;
-  (void) argv;
-  assertf(hts_count_fits(10, 1000) == HTS_TRUE);
-  assertf(hts_count_fits(0, 10) == HTS_TRUE);
-  assertf(hts_count_fits(65535, 10) == HTS_FALSE);
-  assertf(hts_count_fits(1, 0) == HTS_FALSE);
-  assertf(hts_count_fits(1, -1) == HTS_FALSE);
-  printf("java constant-pool cap self-test OK\n");
-  return 0;
-}
-
 /* ------------------------------------------------------------ */
 /* Registry: name -> handler, with a usage hint and a one-line description. */
 /* ------------------------------------------------------------ */
@@ -2461,6 +2605,8 @@ static const struct selftest_entry {
      "memoized vs unmemoized matcher differential", st_filtermemo},
     {"filterdual", "<string1> <string2> <filter>...",
      "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
+    {"filterbounds", "", "matcher length/work caps reject hostile patterns",
+     st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2491,11 +2637,17 @@ static const struct selftest_entry {
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
+    {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
+     st_identabs},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
     {"headerlong", "[header-name:]",
      "over-long header value must not overflow the parse scratch",
      st_headerlong},
+    {"crange", "<raw-content-range-line> ...",
+     "Content-Range parse integer safety", st_crange},
+    {"xfread-limit", "", "in-memory receive buffer size bound",
+     st_xfread_limit},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
@@ -2532,7 +2684,6 @@ static const struct selftest_entry {
      st_ftpline},
 #endif
     {"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
-    {"java", "", "java .class constant-pool count cap self-test", st_java},
 };
 
 static void list_selftests(void) {

src/httrack.c

@@ -919,9 +919,6 @@ static void signal_handlers(void) {
 #ifdef SIGSEGV
   signal(SIGSEGV, sig_fatal);   // segmentation violation
 #endif
-#ifdef SIGSTKFLT
-  signal(SIGSTKFLT, sig_fatal); // stack fault
-#endif
 }
 
 // fin routines de détournement de SIGHUP & co

src/libhttrack.dsp

@@ -224,14 +224,6 @@ SOURCE=htsinthash.c
 # End Source File
 # Begin Source File
 
-SOURCE=htsjava.c
-# End Source File
-# Begin Source File
-
-SOURCE=htsjava.h
-# End Source File
-# Begin Source File
-
 SOURCE=htslib.c
 # End Source File
 # Begin Source File

src/webhttrack

@@ -12,10 +12,9 @@ if test -n "${BROWSER}"; then
 fi
 # Patch for Darwin/Mac by Ross Williams
 if test "$(uname -s)" == "Darwin"; then
-    # Darwin/Mac OS X uses a system 'open' command to find
-    # the default browser. The -W flag causes it to wait for
-    # the browser to exit
-    BROWSEREXE="/usr/bin/open -W"
+    # Darwin's 'open -W' launches the default browser and waits for it to quit.
+    # Keep it only as a fallback so a configured $BROWSER still wins the search.
+    BROWSEREXEFALLBACK="/usr/bin/open -W"
 fi
 BINWD=$(dirname "$0")
 SRCHPATH=("$BINWD" /usr/local/bin /usr/share/bin /usr/bin /usr/lib/httrack /usr/local/lib/httrack /usr/local/share/httrack /opt/local/bin /sw/bin "${HOME}/usr/bin" "${HOME}/bin")
@@ -81,6 +80,7 @@ for i in "${SRCHBROWSEREXE[@]}"; do
     done
     test -n "$BROWSEREXE" && break
 done
+test -n "$BROWSEREXE" || BROWSEREXE="${BROWSEREXEFALLBACK}"
 test -n "$BROWSEREXE" || ! log "Could not find any suitable browser" || exit 1
 
 # "browse" command
@@ -132,12 +132,12 @@ function cleanup {
 }
 
 # Cleanup in case of emergency
-trap "cleanup now; exit" HUP INT QUIT ILL TRAP ABRT BUS FPE SEGV PIPE ALRM TERM STKFLT XCPU XFSZ
+trap "cleanup now; exit" HUP INT QUIT PIPE TERM
 
 # Got SRVURL, launch browser
 launch_browser "${BROWSEREXE}" "${SRVURL}"
 
 # That's all, folks!
-trap "" HUP INT QUIT ILL TRAP ABRT BUS FPE SEGV PIPE ALRM TERM STKFLT XCPU XFSZ
+trap "" HUP INT QUIT PIPE TERM
 cleanup
 exit 0

tests/01_engine-crange.test

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Content-Range parse (treathead via -#test=crange). Hostile values must clamp
+# to 0, and the crange +/- 1 arithmetic must not sign-overflow (UBSan aborts on
+# the pre-fix binary at the INT64_MIN cases).
+
+cr() {
+    local want="$1"
+    shift
+    out="$(httrack -O /dev/null -#test=crange "$@")"
+    test "$out" == "$want" || {
+        echo "FAIL: $* -> '$out' (want '$want')"
+        exit 1
+    }
+}
+
+# Normal range and the '*/N' fallback both parse through unchanged.
+cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes 0-70870/70871'
+cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes */70871'
+
+# INT64_MIN total in the fallback would make crange-1 overflow: clamp to 0.
+cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes */-9223372036854775808'
+
+# Any negative field in the 3-field form clamps the whole triple to 0 (each
+# field independently: a leading negative start/end alone still zeroes all).
+cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -1--1/-1'
+cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -5-10/20'
+
+# INT64_MAX is non-negative: it passes through, the parse must not mangle it.
+cr 'crange_start=0 crange_end=9223372036854775807 crange=9223372036854775807' \
+    'Content-Range: bytes 0-9223372036854775807/9223372036854775807'

tests/01_engine-filter.test

@@ -192,3 +192,6 @@ test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$su
 # (guards against a single-backtrack-point matcher rewrite)
 match '*[aX]X*[a]Y' 'aXaXaY'
 nomatch '*[aX]X*[a]Y' 'aXbXaY'
+
+# hostile patterns must not stack-overflow or hang: length + work caps
+test "$(with_timeout httrack -O /dev/null -#test=filterbounds)" == "filterbounds: OK" || exit 1

tests/01_engine-identabs.test

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# A 2047-byte hostless "?"-URL used to overflow fil[] by one byte and abort in
+# strncat_safe_; the guard now returns -1 while valid URLs still parse.
+test "$(httrack -O /dev/null -#test=identabs)" == "identabs self-test OK" || exit 1

tests/01_engine-java.test

@@ -1,7 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# .class constant-pool count is capped to the file size (calloc DoS).
-httrack -O /dev/null -#test=java run | grep -q "java constant-pool cap self-test OK"

tests/01_engine-xfread.test

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# http_xfread1 must refuse an in-memory receive buffer that would exceed a
+# 32-bit index (hostile Content-Length, or an endless stream) rather than
+# allocate it, while still accepting a normal small size. -#test=xfread-limit
+# drives both refusal paths and the accept path.
+
+out="$(httrack -O /dev/null -#test=xfread-limit)"
+for case in bylen bygrow; do
+    echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
+        echo "FAIL ${case}: $out"
+        exit 1
+    }
+done
+
+# Exactly INT32_MAX must be refused too (the reallocs add 1).
+echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
+    echo "FAIL boundary: $out"
+    exit 1
+}
+
+# The guard must NOT fire for a legitimate small size.
+if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
+    echo "FAIL accept (guard fired on a legit size): $out"
+    exit 1
+fi

tests/19_local-connect-fallback.test

@@ -66,9 +66,12 @@ test -n "$port" || {
 }
 
 out="$tmpdir/crawl"
+# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
+# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
+# bound the stall without changing what the fallback exercises.
 HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
     httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
-    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
+    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
 
 log="$out/hts-log.txt"
 
@@ -92,12 +95,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
     exit 1
 }
 
-# every address refused: the slot exhausts the list, then errors out (the
-# harness timeout would catch a hang/loop; refused connects are instant)
+# every address dead: the slot exhausts the list, then errors out (the harness
+# timeout would catch a hang/loop)
 out2="$tmpdir/crawl2"
 HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
     httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
-    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
+    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
 log2="$out2/hts-log.txt"
 
 grep -q "trying next address" "$log2" || {

tests/31_local-javaclass.test

@@ -1,23 +0,0 @@
-#!/bin/bash
-# The java plugin must load (versioned dlopen name) and parse a .class
-# constant pool: a resource named only inside Foo.class gets crawled.
-set -e
-
-: "${top_srcdir:=..}"
-
-tmproot=$(mktemp -d)
-trap 'rm -rf "$tmproot"' EXIT
-mkdir "$tmproot/javaclass"
-
-cat >"$tmproot/javaclass/index.html" <<'EOF'
-<html><body><a href="Foo.class">applet</a></body></html>
-EOF
-printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
-# magic/minor/major, count=2, one CONSTANT_Utf8 "hello.gif", class/superclass
-printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
-    >"$tmproot/javaclass/Foo.class"
-
-bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
-    --found 'javaclass/Foo.class' \
-    --found 'javaclass/hello.gif' \
-    httrack 'BASEURL/javaclass/index.html'

tests/36_local-bigcrawl.test

@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors 4 --files 361 \
+    --errors-content 4 --files 361 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \
@@ -53,4 +53,4 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --log-found ', no files updated' \
     --max-mirror-bytes 700000 \
     --min-mirror-bytes 500000 \
-    httrack 'BASEURL/big/index.html' --retries=0 -c8 -%c100 -A100000000
+    httrack 'BASEURL/big/index.html' --retries=0 -c16 -%c100 -A100000000

tests/42_local-maxsize-slow.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# -M byte cap under a slow server (#77): p0 overruns -M at once while p1..p3
+# trickle for a minute; the cap must abort the in-flight transfers, not wait
+# them out. Unfixed, the smooth stop drains the trickle at server pace.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+start=$(date +%s)
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    httrack 'BASEURL/bigtrickle/index.html' -M400000 -c4
+wall=$(($(date +%s) - start))
+# trickle runs 60s; a smooth-only stop waits it out, the hard stop lands near 8s.
+# Wall clock is the discriminating check: the log line above fires either way.
+if [ "$wall" -ge 30 ]; then
+    echo "crawl took ${wall}s, -M hard stop did not engage" >&2
+    exit 1
+fi

tests/43_local-update-truncate.test

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# -M hard-abort must not destroy already-complete files (#77 follow-up).
+# Pass 1 mirrors bigtrunc fully. Pass 2 re-fetches under --update -M: fast.bin
+# overruns the cap, its abort tears down slow.bin's stalled re-fetch. Both were
+# complete on disk from pass 1, and both must survive the aborted update:
+#   - slow.bin's re-fetch is incomplete; the direct-to-disk write must not be
+#     left truncated (the previous copy is restored).
+#   - fast.bin fully arrives but is torn down; the aborted slot must not overwrite
+#     it with an empty body.
+# Unfixed, slow.bin is truncated and fast.bin is zeroed while the cache still
+# records them complete.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update -M400000' \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --found bigtrunc/slow.bin \
+    --file-min-bytes bigtrunc/slow.bin 655360 \
+    --file-min-bytes bigtrunc/fast.bin 655360 \
+    httrack 'BASEURL/bigtrunc/index.html' -c2

tests/44_local-update-errormask.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
+# keep its good local copy, not be overwritten by the error body nor purged.
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --rerun \
+    --found 'errmask/keep.dat' \
+    --file-min-bytes 'errmask/keep.dat' 1024 \
+    --file-matches 'errmask/keep.dat' '^KEEP' \
+    --file-not-matches 'errmask/keep.dat' 'error 403' \
+    httrack 'BASEURL/errmask/index.html'

tests/45_local-maxsize-recv.test

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# -M caps received volume, not saved 200-only bytes (#520): links to large 404
+# bodies pump received >> saved; the cap must trip though little lands on disk.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# The mirror bound is the load-bearing half: it fires while <100 KB is saved,
+# so the cap can only have tripped on received volume, not saved bytes (which
+# would exceed the cap and fetch all 16 links).
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 500000 bytes have been transferred.. giving up' \
+    --max-mirror-bytes 100000 \
+    httrack 'BASEURL/maxrecv/index.html' -M500000 -c4

tests/46_local-update-304-resume.test

@@ -0,0 +1,118 @@
+#!/bin/bash
+# C7: a server 304 answering a resume's Range request is out of protocol; httrack
+# must drop the partial and refetch, not finalize it as complete at the partial
+# byte count. Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets a bogus
+# 304, and must recover the whole file.
+set -u
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+server="${testdir}/local-server.py"
+
+command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
+serverpid=
+crawlpid=
+cleanup() {
+    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
+    if test -n "$serverpid"; then
+        kill "$serverpid" 2>/dev/null
+        wait "$serverpid" 2>/dev/null
+    fi
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+counter="${tmpdir}/reqcount"
+mark="${tmpdir}/got304"
+RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+base="http://127.0.0.1:${port}"
+
+which httrack >/dev/null || {
+    echo "could not find httrack"
+    exit 1
+}
+out="${tmpdir}/crawl"
+mkdir "$out"
+common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
+refdir="${out}/hts-cache/ref"
+
+# --- pass 1: crawl, interrupt once the blob download is underway -------------
+printf '[pass 1: interrupt mid-download] ..\t'
+httrack "${common[@]}" "${base}/resume304/index.html" >"${tmpdir}/log1" 2>&1 &
+crawlpid=$!
+for _ in $(seq 1 300); do
+    test -s "$counter" && break
+    kill -0 "$crawlpid" 2>/dev/null || break
+    sleep 0.1
+done
+sleep 0.5
+kill -TERM "$crawlpid" 2>/dev/null
+wait "$crawlpid" 2>/dev/null
+crawlpid=
+test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
+    echo "FAIL: no temp-ref survived pass 1; cannot drive C7"
+    exit 1
+}
+echo "OK (temp-ref present)"
+before=$(wc -c <"$counter" 2>/dev/null || echo 0)
+
+# --- pass 2: --continue -> resume Range -> bogus 304 -> drop + refetch --------
+printf '[pass 2: resume gets 304, must refetch] ..\t'
+httrack "${common[@]}" --continue "${base}/resume304/index.html" >"${tmpdir}/log2" 2>&1
+echo "OK (terminated)"
+
+blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
+full=8200 # len(RESUME304_BODY) = len("C7DATA--") + 8192
+
+printf '[file recovered whole] ..\t'
+test -s "$blob" || {
+    echo "FAIL: blob.bin missing after the 304 resume (partial dropped, not refetched)"
+    exit 1
+}
+got=$(wc -c <"$blob")
+test "$got" -eq "$full" || {
+    echo "FAIL: blob.bin is ${got} bytes, expected ${full} (partial finalized as complete)"
+    exit 1
+}
+echo "OK (${got} bytes)"
+
+# Pass 2 must issue at least two requests: the resume (Range -> 304) and the
+# range-less refetch (-> 200). Exactly one means the partial was trusted.
+after=$(wc -c <"$counter" 2>/dev/null || echo 0)
+hits=$((after - before))
+printf '[resume then refetch] ..\t'
+test "$hits" -ge 2 || {
+    echo "FAIL: only ${hits} pass-2 request(s); the 304 was trusted, no refetch"
+    exit 1
+}
+echo "OK (${hits} requests)"
+
+# The recovery must go through the resume path: a Range request that drew the
+# bogus 304 (a fresh re-crawl sends no Range, so this marker stays empty).
+printf '[resume Range drew a 304] ..\t'
+test -s "$mark" || {
+    echo "FAIL: no Range request got a 304; the resume path was not exercised"
+    exit 1
+}
+echo "OK"

tests/47_local-crange-overflow.test

@@ -0,0 +1,96 @@
+#!/bin/bash
+# C2: a resume whose 206 carries a Content-Range end of INT64_MAX must not
+# sign-overflow the crange+1 range check (UBSan aborts on the pre-fix binary).
+# Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets the hostile 206, and
+# must reject the range and refetch the whole file rather than overflow. Teeth
+# are UBSan-only: a normal build wraps to the same reject+restart.
+set -u
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+server="${testdir}/local-server.py"
+
+command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crange.XXXXXX") || exit 1
+serverpid=
+crawlpid=
+cleanup() {
+    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
+    if test -n "$serverpid"; then
+        kill "$serverpid" 2>/dev/null
+        wait "$serverpid" 2>/dev/null
+    fi
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+base="http://127.0.0.1:${port}"
+
+which httrack >/dev/null || {
+    echo "could not find httrack"
+    exit 1
+}
+out="${tmpdir}/crawl"
+mkdir "$out"
+common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
+refdir="${out}/hts-cache/ref"
+
+# --- pass 1: crawl, interrupt once the blob download is underway -------------
+printf '[pass 1: interrupt mid-download] ..\t'
+httrack "${common[@]}" "${base}/crange206/index.html" >"${tmpdir}/log1" 2>&1 &
+crawlpid=$!
+for _ in $(seq 1 300); do
+    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
+    kill -0 "$crawlpid" 2>/dev/null || break
+    sleep 0.1
+done
+sleep 0.3
+kill -TERM "$crawlpid" 2>/dev/null
+wait "$crawlpid" 2>/dev/null
+crawlpid=
+test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
+    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
+    exit 1
+}
+echo "OK (temp-ref present)"
+
+# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
+printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
+httrack "${common[@]}" --continue "${base}/crange206/index.html" >"${tmpdir}/log2" 2>&1
+echo "OK (terminated)"
+
+blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
+full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
+
+printf '[file recovered whole] ..\t'
+test -s "$blob" || {
+    echo "FAIL: blob.bin missing after the hostile 206"
+    exit 1
+}
+got=$(wc -c <"$blob")
+test "$got" -eq "$full" || {
+    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
+    exit 1
+}
+echo "OK (${got} bytes)"

tests/48_local-crange-memresume.test

@@ -0,0 +1,96 @@
+#!/bin/bash
+# C2 (memory branch): a resume whose 206 lies text/html with a matching
+# INT64_MAX Content-Length must not overflow the resume buffer-size add
+# (UBSan aborts on the pre-fix binary at alloc_mem += totalsize). Pass 1 leaves
+# a partial + temp-ref; pass 2 resumes and must reject and refetch the whole
+# file. Teeth are UBSan-only: a normal build wraps to the same reject+restart.
+set -u
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+server="${testdir}/local-server.py"
+
+command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangemem.XXXXXX") || exit 1
+serverpid=
+crawlpid=
+cleanup() {
+    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
+    if test -n "$serverpid"; then
+        kill "$serverpid" 2>/dev/null
+        wait "$serverpid" 2>/dev/null
+    fi
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+base="http://127.0.0.1:${port}"
+
+which httrack >/dev/null || {
+    echo "could not find httrack"
+    exit 1
+}
+out="${tmpdir}/crawl"
+mkdir "$out"
+common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
+refdir="${out}/hts-cache/ref"
+
+# --- pass 1: crawl, interrupt once the blob download is underway -------------
+printf '[pass 1: interrupt mid-download] ..\t'
+httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
+crawlpid=$!
+for _ in $(seq 1 300); do
+    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
+    kill -0 "$crawlpid" 2>/dev/null || break
+    sleep 0.1
+done
+sleep 0.3
+kill -TERM "$crawlpid" 2>/dev/null
+wait "$crawlpid" 2>/dev/null
+crawlpid=
+test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
+    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
+    exit 1
+}
+echo "OK (temp-ref present)"
+
+# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
+printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
+httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1
+echo "OK (terminated)"
+
+blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
+full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
+
+printf '[file recovered whole] ..\t'
+test -s "$blob" || {
+    echo "FAIL: blob.bin missing after the hostile 206"
+    exit 1
+}
+got=$(wc -c <"$blob")
+test "$got" -eq "$full" || {
+    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
+    exit 1
+}
+echo "OK (${got} bytes)"

tests/49_local-cookiewall.test

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Cookie-wall self-redirect (#15): httrack replays a Set-Cookie to mirror the
+# real page, gives up when nothing changes, and stays bounded when it never does.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# Unknown-ext (wall.php) and known-ext (wall.html): the wall sets the cookie once,
+# then serves real content; httrack must replay it and mirror that content.
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --found 'cookiewall/wall.html' \
+    --file-matches 'cookiewall/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
+    httrack 'BASEURL/cookiewall/index.html'
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --found 'cookiewall2/wall.html' \
+    --file-matches 'cookiewall2/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
+    httrack 'BASEURL/cookiewall2/index.html'
+
+# No Set-Cookie: the jar never changes, so httrack must give up at once (retry
+# stays gated) and not mirror the wall. A dropped gate would skip this log line.
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --not-found 'cookiewall3/wall.html' \
+    --log-found 'loop to same filename' \
+    httrack 'BASEURL/cookiewall3/index.html'
+
+# Ever-changing cookie never satisfies the wall: httrack must stop at the retry
+# cap (bounded, no early give-up) and not mirror it.
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --not-found 'cookiewall4/wall.html' \
+    --log-not-found 'loop to same filename' \
+    httrack 'BASEURL/cookiewall4/index.html'

tests/Makefile.am

@@ -30,6 +30,7 @@ TESTS = \
 	01_engine-cmdline.test \
 	01_engine-cookies.test \
 	01_engine-copyopt.test \
+	01_engine-crange.test \
 	01_engine-dns.test \
 	01_engine-doitlog.test \
 	01_engine-entities.test \
@@ -43,9 +44,9 @@ TESTS = \
 	01_engine-header.test \
 	01_engine-idna.test \
 	01_engine-identurl.test \
+	01_engine-identabs.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
-	01_engine-java.test \
 	01_engine-makeindex.test \
 	01_engine-mime.test \
 	01_engine-parse.test \
@@ -65,6 +66,7 @@ TESTS = \
 	01_engine-urlhack.test \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
+	01_engine-xfread.test \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
@@ -100,7 +102,6 @@ TESTS = \
 	28_local-pause.test \
 	29_local-redirect-fragment.test \
 	30_local-fragment-link.test \
-	31_local-javaclass.test \
 	32_local-cdispo.test \
 	33_local-delayed.test \
 	34_local-maxtime.test \
@@ -110,6 +111,14 @@ TESTS = \
 	38_local-update-304.test \
 	39_local-delayed-cancel.test \
 	40_local-why.test \
-	41_local-utf8-link.test
+	41_local-utf8-link.test \
+	42_local-maxsize-slow.test \
+	43_local-update-truncate.test \
+	44_local-update-errormask.test \
+	45_local-maxsize-recv.test \
+	46_local-update-304-resume.test \
+	47_local-crange-overflow.test \
+	48_local-crange-memresume.test \
+	49_local-cookiewall.test
 
 CLEANFILES = check-network_sh.cache

tests/crawl-test.sh

@@ -105,7 +105,7 @@ function start-crawl {
     log="${tmp}/log"
     debug starting httrack -O "${tmp}" "${moreargs[@]}" "${@:pos}"
     info "running httrack ${*:pos}"
-    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
+    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -mrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
     crawlpid="$!"
     debug "started cralwer on pid $crawlpid"
     wait "$crawlpid"

tests/local-crawl.sh

@@ -13,15 +13,21 @@
 #
 # Usage:
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
-#       --errors N --files N --found PATH ... --directory PATH ... \
+#       [--rerun-args 'ARGS'] \
+#       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
-#       --max-mirror-bytes N \
+#       --file-min-bytes PATH N --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
+# --errors counts every "Error:" log line; --errors-content drops transient
+# network failures (codes -2..-6) that flake on busy loopback under -c8.
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
+# --file-min-bytes asserts a mirrored file (PATH) is at least N bytes.
+# --rerun-args runs a second pass (same server and mirror dir) with the given
+# extra httrack args appended, e.g. an --update run under a cap.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
@@ -39,6 +45,7 @@ key="${testdir}/server.key"
 tls=
 verbose=
 rerun=
+rerun_args=
 rerun_dead=
 tmpdir=
 serverpid=
@@ -122,7 +129,11 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
-    --errors | --files)
+    --rerun-args)
+        pos=$((pos + 1))
+        rerun_args="${args[$pos]}"
+        ;;
+    --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -130,7 +141,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --file-matches | --file-not-matches)
+    --file-matches | --file-not-matches | --file-min-bytes)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
         pos=$((pos + 2))
         ;;
@@ -202,7 +213,7 @@ mkdir "$out" || die "could not create $out"
 declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
 log="${tmpdir}/log"
 info "running httrack ${hts[*]}"
-httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
 crawlpid=$!
 wait "$crawlpid"
 crawlres=$?
@@ -219,7 +230,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -241,6 +252,25 @@ if test -n "$rerun"; then
     fi
 fi
 
+# --- optional second pass with extra args (e.g. an --update run under a cap) ---
+# Same server and mirror dir as the first pass, so the second pass sees the
+# cache the first pass wrote. Used to exercise re-fetch/update behaviour.
+if test -n "$rerun_args"; then
+    read -ra extra <<<"$rerun_args"
+    info "re-running httrack with ${rerun_args}"
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+        "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
+    crawlpid=$!
+    wait "$crawlpid"
+    crawlres=$?
+    crawlpid=
+    test "$crawlres" -eq 0 || ! result "second pass exited $crawlres" || {
+        cat "${log}.2" >&2
+        exit 1
+    }
+    result "OK (second pass)"
+fi
+
 # --- optional dead pass: server stopped, the cache must survive the rollback --
 if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
@@ -251,7 +281,7 @@ if test -n "$rerun_dead"; then
     wait "$serverpid" 2>/dev/null
     serverpid=
     info "re-running httrack against the stopped server"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
     crawlpid=$!
     wait "$crawlpid" || true
@@ -306,6 +336,14 @@ while test "$i" -lt "${#audit[@]}"; do
         assert_equals "checking errors" "${audit[$i]}" \
             "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
         ;;
+    --errors-content)
+        i=$((i + 1))
+        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
+        # transient network failures (statuscode -2..-6) flake on busy loopback;
+        # the code parens are followed by " at link" or " after N retries at link"
+        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
+        assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
+        ;;
     --files)
         i=$((i + 1))
         nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
@@ -388,6 +426,16 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
+    --file-min-bytes)
+        path="${audit[$((i + 1))]}"
+        i=$((i + 2))
+        sz=$(wc -c <"${hostroot}/${path}" 2>/dev/null | tr -d '[:space:]')
+        info "checking ${path} size ${sz:-0} >= ${audit[$i]} bytes"
+        if test -n "$sz" && test "$sz" -ge "${audit[$i]}"; then result "OK"; else
+            result "file too small (or missing)"
+            exit 1
+        fi
+        ;;
     esac
     i=$((i + 1))
 done

tests/local-server.py

@@ -790,6 +790,56 @@ class Handler(SimpleHTTPRequestHandler):
         self.send_header("Content-Length", "0")
         self.end_headers()
 
+    # C7: stall the first GET (partial + temp-ref), then answer the resume's
+    # Range with a bogus 304; httrack must drop the partial and refetch.
+    RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
+    _resume304_started = False
+
+    def route_resume304_index(self):
+        self.send_html('\t<a href="blob.bin">blob</a>')
+
+    def route_resume304(self):
+        counter = os.environ.get("RESUME304_COUNTER")
+        if counter:
+            with open(counter, "a") as fp:
+                fp.write("x")
+        rng = self.headers.get("Range")
+        if not Handler._resume304_started:
+            Handler._resume304_started = True
+            self.send_response(200)
+            self.send_header("Content-Type", "application/octet-stream")
+            self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
+            self.send_header("Accept-Ranges", "bytes")
+            self.send_header("Last-Modified", BIG_LASTMOD)
+            self.send_header("ETag", '"c7"')
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.RESUME304_BODY[:4096])
+                self.wfile.flush()
+                try:
+                    while True:
+                        time.sleep(3600)
+                except OSError:
+                    pass
+            return
+        if rng is not None:  # resume request: bogus out-of-protocol 304
+            mark = os.environ.get("RESUME304_MARK")
+            if mark:
+                with open(mark, "a") as fp:
+                    fp.write("z")
+            self.send_response(304)
+            self.send_header("ETag", '"c7"')
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+            return
+        # Range-less refetch after the partial is dropped: whole file.
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.RESUME304_BODY)
+
     # 206 resume must honor the server's Content-Range, not the offset we asked
     # for (#198): a server resuming a few bytes *before* the request must not
     # leave httrack duplicating the overlap onto the partial. flaky.bin
@@ -855,6 +905,94 @@ class Handler(SimpleHTTPRequestHandler):
         if self.command != "HEAD":
             self.wfile.write(part)
 
+    # C2: a resume answered with a 206 whose Content-Range end is INT64_MAX would
+    # sign-overflow the crange+1 range check (UBSan abort). Stall first (partial +
+    # ref), then answer the resume Range with that hostile 206; httrack must reject
+    # the range and refetch, never overflow.
+    CRANGE206_BODY = b"CR206DAT" + bytes((i * 5 + 1) % 256 for i in range(6000))
+    _crange206_started = False
+
+    def route_crange206_index(self):
+        self.send_html('\t<a href="blob.bin">blob</a>')
+
+    def route_crange206(self):
+        rng = self.headers.get("Range")
+        if not Handler._crange206_started:
+            Handler._crange206_started = True
+            self.send_response(200)
+            self.send_header("Content-Type", "application/octet-stream")
+            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
+            self.send_header("Accept-Ranges", "bytes")
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.CRANGE206_BODY[:3000])
+                self.wfile.flush()
+                try:
+                    while True:
+                        time.sleep(3600)
+                except OSError:
+                    pass
+            return
+        if rng is not None:  # resume: hostile 206, Content-Range end = INT64_MAX
+            self.send_response(206, "Partial Content")
+            self.send_header("Content-Type", "application/octet-stream")
+            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
+            self.send_header("Content-Range", "bytes 0-9223372036854775807/1")
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.CRANGE206_BODY)
+            return
+        # range-less refetch after the bad range is rejected: whole file.
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.CRANGE206_BODY)
+
+    # C2 memory branch: a resume answered with a 206 that lies text/html (so the
+    # resume buffers in memory) plus a matching INT64_MAX Content-Length would
+    # overflow the buffer-size add. Stall first, then send that hostile 206.
+    _crange206mem_started = False
+
+    def route_crange206mem_index(self):
+        self.send_html('\t<a href="blob.bin">blob</a>')
+
+    def route_crange206mem(self):
+        rng = self.headers.get("Range")
+        if not Handler._crange206mem_started:
+            Handler._crange206mem_started = True
+            self.send_response(200)
+            self.send_header("Content-Type", "application/octet-stream")
+            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
+            self.send_header("Accept-Ranges", "bytes")
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.CRANGE206_BODY[:3000])
+                self.wfile.flush()
+                try:
+                    while True:
+                        time.sleep(3600)
+                except OSError:
+                    pass
+            return
+        if rng is not None:  # resume: text/html + matching INT64_MAX Content-Length
+            self.send_response(206, "Partial Content")
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", "9223372036854775807")
+            self.send_header(
+                "Content-Range", "bytes 0-9223372036854775806/9223372036854775807"
+            )
+            self.end_headers()
+            return  # the overflow is computed before any body read
+        # range-less refetch after the resume is rejected: whole file.
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.CRANGE206_BODY)
+
     # error pages / 0-byte files (#17): -o0 ("no error pages") must keep 4xx/5xx
     # bodies off disk; a genuine 0-byte 200 is a valid file and stays.
     def route_errpage_index(self):
@@ -934,6 +1072,33 @@ class Handler(SimpleHTTPRequestHandler):
     def route_mini304_page(self):
         self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
 
+    # --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
+    # on the update fetch must keep its good copy, not be overwritten nor purged.
+    ERRMASK_GOOD = b"KEEP" + b"." * 1020  # 1024 B distinctive non-HTML body
+    ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
+
+    def route_errmask_index(self):
+        self.send_html('\t<a href="keep.dat">keep</a>\n')
+
+    def route_errmask_keep(self):
+        # First crawl (no validator) gets the 1024 B body + Last-Modified; the
+        # update sends a conditional and gets a 403 error page.
+        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
+            self.send_response(403, "Forbidden")
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.ERRMASK_ERR)
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Last-Modified", BIG_LASTMOD)
+        self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.ERRMASK_GOOD)
+
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -978,6 +1143,56 @@ class Handler(SimpleHTTPRequestHandler):
     def route_delayed_empty(self):
         self.send_raw(b"", "text/html")  # 200 + Content-Length: 0
 
+    # --- /cookiewall/ (#15): a self-redirect that only sets a cookie is a
+    # consent wall; httrack must replay the cookie and fetch the real page.
+    WALL_MARK = b"REAL-CONTENT-BEHIND-COOKIE-WALL"
+
+    def route_cookiewall_index(self):
+        self.send_html('\t<a href="wall.php">wall</a>')
+
+    def route_cookiewall_wall(self):
+        self._cookiewall_reply("wall.php")
+
+    # Known-extension twin: .html so the type is not delayed-resolved.
+    def route_cookiewall2_index(self):
+        self.send_html('\t<a href="wall.html">wall</a>')
+
+    def route_cookiewall2_wall(self):
+        self._cookiewall_reply("wall.html")
+
+    def _cookiewall_reply(self, location):
+        if self.request_cookies().get("gate") == "1":
+            self.send_raw(
+                b"<html><body>" + self.WALL_MARK + b"</body></html>\n", "text/html"
+            )
+        else:
+            self._wall_redirect(location, "gate=1; Path=/")
+
+    # No-cookie self-redirect: the jar never changes, so httrack must give up at
+    # once rather than re-fetch (proves the cookie-wall retry stays gated on #15).
+    def route_cookiewall3_index(self):
+        self.send_html('\t<a href="wall.php">wall</a>')
+
+    def route_cookiewall3_wall(self):
+        self._wall_redirect("wall.php", None)
+
+    # Ever-changing cookie: every hit sets a fresh value, so the jar keeps
+    # changing; httrack must stop at the loops<7 cap, not spin forever.
+    def route_cookiewall4_index(self):
+        self.send_html('\t<a href="wall.php">wall</a>')
+
+    def route_cookiewall4_wall(self):
+        nonce = int(self.request_cookies().get("gate", "0")) + 1
+        self._wall_redirect("wall.php", f"gate={nonce}; Path=/")
+
+    def _wall_redirect(self, location, set_cookie):
+        self.send_response(302, "Found")
+        self.send_header("Location", location)
+        if set_cookie is not None:
+            self.send_header("Set-Cookie", set_cookie)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
     # -E time-limit (#481): pages that trickle far longer than any -E budget,
     # so only an engine-side abort can end the crawl.
     TRICKLE_SECONDS = 60
@@ -1035,6 +1250,64 @@ class Handler(SimpleHTTPRequestHandler):
     def route_bigfile(self):
         self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
 
+    # -M under a slow server (#77): p0 is a fast 640KB file that alone overruns
+    # -M; p1..p3 trickle for a minute. The cap must abort those in-flight
+    # transfers, not wait them out.
+    def route_bigtrickle_index(self):
+        self.send_html(
+            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(4))
+        )
+
+    # -M hard-abort must not destroy an already-complete file (#77 follow-up).
+    # "fast.bin" alone overruns -M and completes; "slow.bin" transfers fully on
+    # its first fetch (initial mirror) but stalls on every later fetch (the
+    # --update re-fetch), so the -M abort tears it down mid-body. An engine that
+    # truncates the good local copy on the aborted re-fetch loses data.
+    slow_seen = 0
+
+    def route_bigtrunc_index(self):
+        self.send_html('\t<a href="fast.bin">fast</a>\n\t<a href="slow.bin">slow</a>\n')
+
+    def route_bigtrunc_slow(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(self.BIGFILE_BYTES))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        # Count body fetches only, so a stray HEAD can't shift which pass stalls.
+        Handler.slow_seen += 1
+        first = Handler.slow_seen == 1
+        try:
+            if first:
+                self.wfile.write(b"x" * self.BIGFILE_BYTES)
+                self.wfile.flush()
+            else:
+                self.wfile.write(b"x" * 4096)
+                self.wfile.flush()
+                for _ in range(120):
+                    self.wfile.write(b"x")
+                    self.wfile.flush()
+                    time.sleep(1.0)
+        except OSError:
+            pass
+
+    # -M received-volume cap (#520): links to large 404 bodies. httrack receives
+    # each (HTS_TOTAL_RECV climbs) but saves none, so saved stays far below -M.
+    def route_maxrecv_index(self):
+        self.send_html(
+            "".join('\t<a href="r%d.bin">r%d</a>\n' % (i, i) for i in range(16))
+        )
+
+    def route_maxrecv_404(self):
+        body = b"x" * self.BIGFILE_BYTES
+        self.send_response(404, "Not Found")
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(body)
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1064,9 +1337,15 @@ class Handler(SimpleHTTPRequestHandler):
         "/intl/" + INTL_NAME: route_intl_page,
         "/resume/index.html": route_resume_index,
         "/resume/blob.txt": route_resume,
+        "/resume304/index.html": route_resume304_index,
+        "/resume304/blob.bin": route_resume304,
         "/overlap/index.html": route_overlap_index,
         "/overlap/flaky.bin": route_overlap,
         "/overlap/full.bin": route_overlap_full,
+        "/crange206/index.html": route_crange206_index,
+        "/crange206/blob.bin": route_crange206,
+        "/crange206mem/index.html": route_crange206mem_index,
+        "/crange206mem/blob.bin": route_crange206mem,
         "/size/index.html": route_size_index,
         "/size/oversize.bin": route_size_oversize,
         "/errpage/index.html": route_errpage_index,
@@ -1107,6 +1386,14 @@ class Handler(SimpleHTTPRequestHandler):
         "/bigfiles/p5.bin": route_bigfile,
         "/bigfiles/p6.bin": route_bigfile,
         "/bigfiles/p7.bin": route_bigfile,
+        "/bigtrickle/index.html": route_bigtrickle_index,
+        "/bigtrickle/p0.bin": route_bigfile,
+        "/bigtrickle/p1.bin": route_trickle_page,
+        "/bigtrickle/p2.bin": route_trickle_page,
+        "/bigtrickle/p3.bin": route_trickle_page,
+        "/bigtrunc/index.html": route_bigtrunc_index,
+        "/bigtrunc/fast.bin": route_bigfile,
+        "/bigtrunc/slow.bin": route_bigtrunc_slow,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,
@@ -1122,11 +1409,38 @@ class Handler(SimpleHTTPRequestHandler):
         "/delayed/chain7.php": route_delayed_chain,
         "/delayed/chain8.php": route_delayed_chain,
         "/delayed/chain9.php": route_delayed_chain,
+        "/cookiewall/index.html": route_cookiewall_index,
+        "/cookiewall/wall.php": route_cookiewall_wall,
+        "/cookiewall2/index.html": route_cookiewall2_index,
+        "/cookiewall2/wall.html": route_cookiewall2_wall,
+        "/cookiewall3/index.html": route_cookiewall3_index,
+        "/cookiewall3/wall.php": route_cookiewall3_wall,
+        "/cookiewall4/index.html": route_cookiewall4_index,
+        "/cookiewall4/wall.php": route_cookiewall4_wall,
         "/redir/index.html": route_redir_index,
         "/redir/go.php": route_redir_go,
         "/redir/target.html": route_redir_target,
         "/mini304/index.html": route_mini304_index,
         "/mini304/page.html": route_mini304_page,
+        "/errmask/index.html": route_errmask_index,
+        "/errmask/keep.dat": route_errmask_keep,
+        "/maxrecv/index.html": route_maxrecv_index,
+        "/maxrecv/r0.bin": route_maxrecv_404,
+        "/maxrecv/r1.bin": route_maxrecv_404,
+        "/maxrecv/r2.bin": route_maxrecv_404,
+        "/maxrecv/r3.bin": route_maxrecv_404,
+        "/maxrecv/r4.bin": route_maxrecv_404,
+        "/maxrecv/r5.bin": route_maxrecv_404,
+        "/maxrecv/r6.bin": route_maxrecv_404,
+        "/maxrecv/r7.bin": route_maxrecv_404,
+        "/maxrecv/r8.bin": route_maxrecv_404,
+        "/maxrecv/r9.bin": route_maxrecv_404,
+        "/maxrecv/r10.bin": route_maxrecv_404,
+        "/maxrecv/r11.bin": route_maxrecv_404,
+        "/maxrecv/r12.bin": route_maxrecv_404,
+        "/maxrecv/r13.bin": route_maxrecv_404,
+        "/maxrecv/r14.bin": route_maxrecv_404,
+        "/maxrecv/r15.bin": route_maxrecv_404,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------
@@ -1331,7 +1645,12 @@ def main():
     def factory(*a, **kw):
         return Handler(*a, directory=root, **kw)
 
-    httpd = ThreadingHTTPServer((args.bind, 0), factory)
+    # macOS/BSD drop SYNs when the listen backlog overflows (Linux is lenient);
+    # raise it from Python's default 5 so a busy -c8 crawl can't lose fetches.
+    class BacklogHTTPServer(ThreadingHTTPServer):
+        request_queue_size = 128
+
+    httpd = BacklogHTTPServer((args.bind, 0), factory)
 
     if args.tls:
         import ssl

tests/webhttrack-smoke.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+# Smoke-test an installed webhttrack: launch it with a stub browser and assert
+# htsserver comes up and serves the web UI. Arg: the install prefix.
+set -euo pipefail
+
+prefix="${1:?usage: webhttrack-smoke.sh <install-prefix>}"
+wht="$prefix/bin/webhttrack"
+test -x "$wht" || {
+    echo "no webhttrack at $wht" >&2
+    exit 1
+}
+
+work="$(mktemp -d)"
+# webhttrack backgrounds htsserver, which outlives it; reap any stray one (scoped
+# to this prefix) so a lingering server can never hold the CI step open.
+trap 'pkill -f "$prefix/bin/htsserver" 2>/dev/null || true; rm -rf "$work"' EXIT
+export HOME="$work/home"
+mkdir -p "$HOME/websites"
+marker="$work/marker"
+
+stubdir="$work/bin"
+mkdir -p "$stubdir"
+
+# On Darwin webhttrack hardcodes "open -W", which launches a real GUI browser and
+# blocks headless. Shadow uname so it takes the generic path and picks the stub
+# browser below; htsserver and webhttrack's path resolution still run for real.
+cat >"$stubdir/uname" <<'EOF'
+#!/bin/bash
+[ "${1:-}" = "-s" ] && {
+    echo Linux
+    exit 0
+}
+exec /usr/bin/uname "$@"
+EOF
+chmod +x "$stubdir/uname"
+
+# Stub browser: webhttrack tries its browser-name list in order and runs the
+# first it finds, so shadow the first entry, "x-www-browser". It fetches the
+# server URL and records PASS only for the working UI: the brand string AND the
+# step-2 form action, which a truncated/degraded template page would lack (the
+# bare title alone is not enough). htsserver only lives until webhttrack exits,
+# so the check has to happen here.
+# -a: the UI is served ISO-8859-1, so grep must not treat it as binary.
+cat >"$stubdir/x-www-browser" <<EOF
+#!/bin/bash
+echo "stub browser invoked with: \$1" >&2
+if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html; then
+    echo PASS >"$marker"
+else
+    echo "FAIL: unexpected response from \$1" >"$marker"
+fi
+EOF
+chmod +x "$stubdir/x-www-browser"
+export PATH="$stubdir:$prefix/bin:$PATH"
+
+echo "launching webhttrack"
+"$wht" </dev/null >"$work/webhttrack.log" 2>&1 &
+whpid=$!
+
+# Bounded poll for the marker (macOS has no timeout(1)); teardown below kills
+# webhttrack and reaps htsserver, so the run is bounded without a watchdog.
+for i in $(seq 1 45); do
+    test -f "$marker" && {
+        echo "marker written after ${i}s"
+        break
+    }
+    kill -0 "$whpid" 2>/dev/null || {
+        echo "webhttrack exited on its own after ${i}s"
+        break
+    }
+    sleep 1
+done
+
+# Reap webhttrack and the htsserver it spawned. Confirm death with a bounded poll
+# (not a blocking wait, which could hang on macOS); SIGKILL if it ignores TERM.
+echo "tearing down"
+kill "$whpid" 2>/dev/null || true
+if pkill -f "$prefix/bin/htsserver" 2>/dev/null; then
+    echo "reaped a lingering htsserver"
+else
+    echo "no lingering htsserver"
+fi
+for _ in $(seq 1 10); do
+    kill -0 "$whpid" 2>/dev/null || break
+    sleep 1
+done
+kill -9 "$whpid" 2>/dev/null || true
+
+echo "--- webhttrack.log ---"
+cat "$work/webhttrack.log" 2>/dev/null || true
+echo "--- end ---"
+echo "marker=[$(cat "$marker" 2>/dev/null || echo NONE)]"
+
+if test "$(cat "$marker" 2>/dev/null || true)" = PASS; then
+    echo "webhttrack smoke: PASS"
+else
+    echo "webhttrack smoke: FAIL" >&2
+    exit 1
+fi