rms/httrack
1
0
Fork 0
mirror of https://github.com/xroche/httrack.git synced 2026-08-04 14:56:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: rms:fix-866-sigaltstack
Branches Tags
rms:master rms:fix-906-relocatable-rpath rms:fix-866-sigaltstack rms:fix-943-installed-header rms:fix-864-js-escaping rms:probe-870-macos15 rms:chore/tests-union-merge rms:singlefile-reuse-htsparse rms:proto/singlefile-placeholder rms:debian-3.49.10-2 rms:staging rms:wiki rms:3.48 rms:3.47
rms:3.49.15 rms:3.49.14 rms:3.49.13 rms:3.49.12 rms:3.49.11 rms:3.49.10 rms:3.49.9 rms:3.49.8-2 rms:3.49.8 rms:3.49.7 rms:3.49.6 rms:3.49.5 rms:3.49.4 rms:3.49.3 rms:3.48.21 rms:3.48.20 rms:3.48.19 rms:3.48.18 rms:3.48.17 rms:3.48.16 rms:3.48.13 rms:3.48.12 rms:3.48.11 rms:3.48.9 rms:3.48.10 rms:3.48.8 rms:3.48.7 rms:3.48.3 rms:3.48.6
..
compare: rms:debian-3.49.10-2
Branches Tags
rms:master rms:fix-906-relocatable-rpath rms:fix-866-sigaltstack rms:fix-943-installed-header rms:fix-864-js-escaping rms:probe-870-macos15 rms:chore/tests-union-merge rms:singlefile-reuse-htsparse rms:proto/singlefile-placeholder rms:debian-3.49.10-2 rms:staging rms:wiki rms:3.48 rms:3.47
rms:3.49.15 rms:3.49.14 rms:3.49.13 rms:3.49.12 rms:3.49.11 rms:3.49.10 rms:3.49.9 rms:3.49.8-2 rms:3.49.8 rms:3.49.7 rms:3.49.6 rms:3.49.5 rms:3.49.4 rms:3.49.3 rms:3.48.21 rms:3.48.20 rms:3.48.19 rms:3.48.18 rms:3.48.17 rms:3.48.16 rms:3.48.13 rms:3.48.12 rms:3.48.11 rms:3.48.9 rms:3.48.10 rms:3.48.8 rms:3.48.7 rms:3.48.3 rms:3.48.6

1 Commits
fix-866-si ... debian-3.4

Author SHA1 Message Date
Xavier Roche
141d065a04 debian: 3.49.10-2, fix buildd FTBFS via quilt patch 
3.49.10-1 failed to build on every Debian buildd: tests/28_local-pause
failed instead of skipping when python3 is absent (the local-server tests
need python3, which the minimal buildd chroot lacks). The fix is already
upstream, but the -1 orig is frozen in the archive, so carry it as a quilt
patch and bump to -2. Packaging-only; no upstream code change.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
 2026-06-28 20:19:47 +02:00
667 changed files with 19762 additions and 53928 deletions
Expand all files Collapse all files

11
.gitattributes vendored
View File

@@ -1,11 +0,0 @@
# Resource scripts are Windows tooling input and must stay CRLF: the engine has no
# encoding guard, so an autocrlf checkout could otherwise flatten them silently.
*.rc text eol=crlf
# The test scripts run under Git Bash on the Windows CI; a CRLF checkout makes
# bash die on $'\r' on every line of them.
*.test text eol=lf
*.sh text eol=lf
# Scoped to this file so a union can never touch build logic.
tests/tests-list.mk merge=union

20
.github/dependabot.yml vendored
View File

@@ -1,20 +0,0 @@
version: 2
updates:
# Bumps builtin-baseline in src/vcpkg.json so the pinned OpenSSL/zlib can't rot.
# windows-build.yml validates each bump; the pin makes what ships reproducible.
- package-ecosystem: vcpkg
directory: /src
schedule:
interval: weekly
# Keep the workflow action pins current (they only rot manually otherwise).
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
# src/coucal is a hard build dependency, and nothing else notices when it moves.
- package-ecosystem: gitsubmodule
directory: /
schedule:
interval: weekly

408
.github/workflows/ci.yml vendored
View File

@@ -31,7 +31,7 @@ jobs:
env:
CC: ${{ matrix.cc }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -41,51 +41,21 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential clang autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: Configure
run: |
set -euo pipefail
# Regenerate: configure and the Makefile.in's are not tracked.
# Regenerate from configure.ac/Makefile.am to validate them; the
# committed generated files already let a plain checkout build.
autoreconf -fi
# Disabling zlib must fail here rather than at link with a pile of
# undefined minizip references (#735). Both spellings, so a rewrite
# cannot keep one and lose the other. Probed out-of-tree to leave
# nothing behind.
nozlib="$RUNNER_TEMP/nozlib"
for arg in --without-zlib --with-zlib=no; do
rm -rf "$nozlib" && mkdir -p "$nozlib"
if (cd "$nozlib" && "$GITHUB_WORKSPACE/configure" "$arg" >out.log 2>&1); then
echo "::error::configure $arg succeeded; it must be rejected"
exit 1
fi
# ... and for the stated reason, not an unrelated configure failure.
grep -q "zlib cannot be disabled" "$nozlib/out.log" \
|| { cat "$nozlib/out.log"; exit 1; }
done
./configure
# Same dead end from the compile side. The bare compile is the
# control: without it a broken probe would pass vacuously.
hdr='#include "htsglobal.h"'
echo "$hdr" | $CC -I. -Isrc -fsyntax-only -xc -
if echo "$hdr" | $CC -DHTS_USEZLIB=0 -I. -Isrc -fsyntax-only -xc - 2>/dev/null; then
echo "::error::-DHTS_USEZLIB=0 compiled; htsglobal.h must reject it"
exit 1
fi
# a missing decoder would silently drop the coding from Accept-Encoding
grep -q "define HTS_USEBROTLI 1" config.h
grep -q "define HTS_USEZSTD 1" config.h
- name: Build
run: make -j"$(nproc)"
# A backstop only: tests/test-timeout.sh bounds each test, and a healthy run
# is a minute here, two on macOS. Without it a stall ran to the job's 6h default.
- name: Test
timeout-minutes: 20
run: |
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
make check -j"$jobs"
run: make check
- name: Print the test log on failure
if: failure()
@@ -100,7 +70,7 @@ jobs:
name: build (no python3, Debian buildd)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -110,7 +80,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: Configure
run: |
@@ -122,7 +92,6 @@ jobs:
run: make -j"$(nproc)"
- name: Test without python3
timeout-minutes: 20
run: |
set -euo pipefail
# Hide every python3* so `command -v python3` fails like it does in the
@@ -130,8 +99,7 @@ jobs:
sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
-exec mv {} {}.hidden \;
! command -v python3
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
make check -j"$jobs"
make check
- name: Print the test log on failure
if: failure()
@@ -143,186 +111,34 @@ jobs:
# so point configure at it; everything else is in the SDK or default paths.
macos:
name: build (macOS arm64, clang)
runs-on: macos-15
runs-on: macos-14
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
brew install autoconf automake libtool autoconf-archive brotli zstd
brew install autoconf automake libtool autoconf-archive
- name: Configure
run: |
set -euo pipefail
ssl="$(brew --prefix openssl@3)"
brewp="$(brew --prefix)"
autoreconf -fi
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
LDFLAGS="-L${ssl}/lib -L${brewp}/lib"
grep -q "define HTS_USEBROTLI 1" config.h
grep -q "define HTS_USEZSTD 1" config.h
./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib"
- name: Build
run: make -j"$(sysctl -n hw.ncpu)"
- name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
# 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
# instantly like Linux; alias them onto lo0 so they don't stall to timeout.
run: |
set -euo pipefail
sudo ifconfig lo0 alias 127.0.0.2 up
sudo ifconfig lo0 alias 127.0.0.3 up
- name: Test
timeout-minutes: 20
run: |
jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
make check -j"$jobs"
run: make check
- name: Print the test log on failure
if: failure()
run: cat tests/test-suite.log 2>/dev/null || true
# Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
# browser path (open -W) and nothing else exercises htsserver. Install into a
# temp prefix, then check webhttrack brings up htsserver and serves the UI.
webhttrack-macos:
name: webhttrack smoke (macOS arm64)
runs-on: macos-15
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
brew install autoconf automake libtool autoconf-archive brotli zstd
- name: Build and install into a temp prefix
run: |
set -euo pipefail
ssl="$(brew --prefix openssl@3)"
brewp="$(brew --prefix)"
autoreconf -fi
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
LDFLAGS="-L${ssl}/lib -L${brewp}/lib" \
--prefix="$RUNNER_TEMP/inst"
make -j"$(sysctl -n hw.ncpu)"
make install
- name: Smoke-test webhttrack
run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
# Same smoke on the platform that ships webhttrack as a package: the install layout
# it asserts (a relative data symlink, #885) is what Debian consumes, and the macOS
# job alone left that untested on Linux.
webhttrack-linux:
name: webhttrack smoke (Linux x86-64)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
- name: Build and install into a temp prefix
run: |
set -euo pipefail
./bootstrap
./configure --prefix="$RUNNER_TEMP/inst"
make -j"$(nproc)"
make install
- name: Smoke-test webhttrack
run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
# The macOS app bundle (#886): assemble it, then prove it still works after being
# moved, which is the only thing a .app has to survive that a prefix install does not.
macos-app:
name: macOS app bundle (arm64)
runs-on: macos-15
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
brew install autoconf automake libtool autoconf-archive brotli zstd
# --disable-shared keeps libhttrack inside the binaries, so the bundle carries no
# absolute dylib path back to the staging prefix.
- name: Build and install into a temp prefix
run: |
set -euo pipefail
ssl="$(brew --prefix openssl@3)"
brewp="$(brew --prefix)"
./bootstrap
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
LDFLAGS="-L${ssl}/lib -L${brewp}/lib" \
--prefix="$RUNNER_TEMP/inst" --disable-shared
make -j"$(sysctl -n hw.ncpu)"
make install
- name: Assemble HTTrack.app
run: |
set -euo pipefail
make macos-app APP_FLAGS="--prefix $RUNNER_TEMP/inst --out $RUNNER_TEMP"
plutil -lint "$RUNNER_TEMP/HTTrack.app/Contents/Info.plist"
- name: Run it through the bundle stub
run: |
set -euo pipefail
app="$RUNNER_TEMP/HTTrack.app"
bash tests/webhttrack-smoke.sh "$app/Contents/Resources" "$app/Contents/MacOS/HTTrack"
- name: Run it again after moving it
run: |
set -euo pipefail
mkdir -p "$RUNNER_TEMP/moved"
mv "$RUNNER_TEMP/HTTrack.app" "$RUNNER_TEMP/moved/"
rm -rf "$RUNNER_TEMP/inst"
app="$RUNNER_TEMP/moved/HTTrack.app"
bash tests/webhttrack-smoke.sh "$app/Contents/Resources" "$app/Contents/MacOS/HTTrack"
# The load-command check cannot fail while the loader can still reach Homebrew (#901).
- name: Run it with Homebrew's libraries out of reach
run: |
set -euo pipefail
app="$RUNNER_TEMP/moved/HTTrack.app"
# Without this the hiding below proves nothing: an empty Frameworks passes it.
ls "$app/Contents/Frameworks"/libssl.*.dylib >/dev/null
# The kegs, not just the opt symlinks: a source build records the Cellar path.
hidden=""
for keg in openssl@3 brotli zstd; do
for d in "$(brew --prefix)/opt/$keg" "$(brew --prefix)/Cellar/$keg"; do
if [ -e "$d" ]; then sudo mv "$d" "$d.hidden"; hidden="$hidden $d"; fi
done
done
trap 'for d in $hidden; do sudo mv "$d.hidden" "$d"; done' EXIT
test -n "$hidden"
bash tests/webhttrack-smoke.sh "$app/Contents/Resources" "$app/Contents/MacOS/HTTrack"
# The release path (macos-release.yml) minus Apple, so a bundle that cannot be
# signed or packed fails on the PR rather than on release day.
- name: Sign and pack the bundle, ad hoc
run: |
set -euo pipefail
bash tools/macos-release.sh --app "$RUNNER_TEMP/moved/HTTrack.app" \
--identity - --skip-notarize
# Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
# -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
# and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
@@ -332,7 +148,7 @@ jobs:
name: build (linux i386, gcc -m32)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -343,8 +159,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential gcc-multilib autoconf automake libtool \
autoconf-archive zlib1g-dev:i386 libssl-dev:i386 \
libbrotli-dev:i386 libzstd-dev:i386
autoconf-archive zlib1g-dev:i386 libssl-dev:i386
- name: Configure
run: |
@@ -356,10 +171,7 @@ jobs:
run: make -j"$(nproc)"
- name: Test
timeout-minutes: 20
run: |
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
make check -j"$jobs"
run: make check
- name: Print the test log on failure
if: failure()
@@ -376,7 +188,7 @@ jobs:
name: sanitize (ASan+UBSan, gcc)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -386,7 +198,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: Configure (sanitized)
run: |
@@ -414,10 +226,7 @@ jobs:
env:
ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
timeout-minutes: 20
run: |
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
make check -j"$jobs"
run: make check
- name: Print the test log on failure
if: failure()
@@ -431,7 +240,7 @@ jobs:
name: msan (MemorySanitizer, clang)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -458,58 +267,16 @@ jobs:
- name: Test (offline self-tests under MSan)
env:
MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
timeout-minutes: 20
run: |
set -euo pipefail
# 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
# skipped here (uninstrumented libz floods MSan with false positives).
tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
make check -j"$jobs" TESTS="$tests"
# Engine self-tests only; the cache trio pulls in uninstrumented zlib.
tests="$(cd tests && ls 01_engine-*.test | grep -v -- '-cache' | tr '\n' ' ')"
make check TESTS="$tests"
- name: Print the test log on failure
if: failure()
run: cat tests/test-suite.log 2>/dev/null || true
# libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
# parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
# (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
# catastrophic backtracking -- and pins the regression seeds for the bugs the
# fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
fuzz:
name: fuzz (libFuzzer corpus replay, clang)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential clang autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
- name: Configure (fuzzers, static)
run: |
set -euo pipefail
autoreconf -fi
./configure CC=clang \
CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
LDFLAGS="-fsanitize=address,undefined" \
--enable-fuzzers --disable-shared
- name: Build
run: make -j"$(nproc)"
- name: Replay corpora
env:
ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
run: bash fuzz/run-fuzzers.sh fuzz check
# Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
# the configuration users on minimal systems build, and one libssl is not even
# installed here so configure cannot silently re-enable it. The matrix above
@@ -519,7 +286,7 @@ jobs:
name: build (no openssl, --disable-https)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -540,45 +307,12 @@ jobs:
run: make -j"$(nproc)"
- name: Test
timeout-minutes: 20
run: |
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
make check -j"$jobs"
run: make check
- name: Print the test log on failure
if: failure()
run: cat tests/test-suite.log 2>/dev/null || true
# OpenSSL 4.0 removes the API 3.x deprecated. Building with that set hidden
# keeps the engine ready for it: a deprecated call fails here rather than the
# day a distro moves. Compile-only, since the headers are what is under test.
openssl-no-deprecated:
name: build (OpenSSL, no deprecated API)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
- name: Configure (deprecated OpenSSL API hidden)
run: |
set -euo pipefail
autoreconf -fi
./configure CPPFLAGS=-DOPENSSL_NO_DEPRECATED
# A build with https off would pass this job vacuously.
grep -q "define HTS_USEOPENSSL 1" config.h
- name: Build
run: make -j"$(nproc)"
# Validate the Debian packaging via the same script maintainers release with.
# One amd64/gcc run is enough: packaging (control/rules/manifest/lintian/quilt
# source build) is arch- and compiler-independent, and the build matrix above
@@ -601,10 +335,10 @@ jobs:
apt-get install -y --no-install-recommends \
ca-certificates git \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev \
zlib1g-dev libssl-dev \
debhelper devscripts lintian fakeroot
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -635,7 +369,7 @@ jobs:
name: distcheck (release tarball)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: recursive
@@ -645,7 +379,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: distcheck
run: |
@@ -660,7 +394,7 @@ jobs:
if: github.event_name == 'pull_request'
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: 0
@@ -699,15 +433,12 @@ jobs:
man/makeman.sh
src/htsbasiccharsets.sh
src/htsentities.sh
src/webhttrack.in
src/webhttrack
tests/*.sh
tests/*.test
tools/macos-app.sh
tools/macos-bundle.sh
tools/macos-release.sh
tools/mkdeb.sh
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install linters
run: |
@@ -715,9 +446,8 @@ jobs:
sudo apt-get update
# noble ships shfmt 3.8.0 (universe), matching the pinned local dev
# version; use it rather than fetching a release binary from github.com.
sudo apt-get install -y --no-install-recommends shellcheck shfmt appstream
sudo apt-get install -y --no-install-recommends shellcheck shfmt
shfmt --version
appstreamcli --version
- name: shellcheck
run: shellcheck $SHELL_SCRIPTS
@@ -725,24 +455,6 @@ jobs:
- name: shfmt
run: shfmt -d -i 4 $SHELL_SCRIPTS
# MSBuild rejects a malformed .vcxproj with a bare MSB4025 and no build, so
# catch it here in seconds rather than on a Windows runner minutes in.
- name: XML well-formedness (MSBuild project files)
run: |
set -euo pipefail
for f in src/*.vcxproj; do
python3 -c "import sys,xml.dom.minidom; xml.dom.minidom.parse(sys.argv[1])" "$f"
echo "ok $f"
done
# A software centre quietly ignores a metainfo it cannot parse. Errors and
# warnings fail; --no-net so the gate is on the file, not httrack.com's uptime.
- name: AppStream metainfo validation
run: |
set -euo pipefail
appstreamcli validate --no-net --explain \
html/server/div/com.httrack.WebHTTrack.metainfo.xml
# Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
# (it was shaped by an old Visual Studio formatter) and does not round-trip,
# so we never reformat the whole tree -- only the lines a PR touches.
@@ -751,7 +463,7 @@ jobs:
if: github.event_name == 'pull_request'
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
fetch-depth: 0
@@ -775,12 +487,7 @@ jobs:
set -euo pipefail
git fetch --no-tags origin \
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
# Merge base, not the branch tip: master moves during a run, and a
# tip-relative diff blames its new C on this PR.
if ! base="$(git merge-base "origin/${{ github.base_ref }}" HEAD)"; then
echo "::error::no merge base with origin/${{ github.base_ref }}; cannot scope the check."
exit 1
fi
base="origin/${{ github.base_ref }}"
set +e
diff="$(git clang-format --binary clang-format-19 --style=file \
--diff --extensions c,h "$base")"
@@ -802,48 +509,3 @@ jobs:
echo "Fix locally with: git clang-format --binary clang-format-19 $base"
exit 1 ;;
esac
docs:
name: documentation (chrome in sync, links resolve)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
# The pages carry their navigation as real markup, because they are read from
# disk as often as over http and file:// has no includes. Regenerating has to
# be a no-op, or a page has drifted from tools/doc-chrome.py.
- name: Shared chrome is in sync
run: python3 tools/doc-chrome.py --check
- name: Relative links and anchors resolve
run: python3 tools/doc-links.py html
man-page-sync:
name: man page / html in sync
if: github.event_name == 'pull_request'
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
# html/httrack.man.html is groff-rendered from man/httrack.1 and committed.
# Rendering needs the full groff html device, so CI can't regenerate it;
# instead require the two to move together: a PR that touches httrack.1
# must also touch the html, catching the "regenerated roff, forgot html".
- name: httrack.1 changes must include html/httrack.man.html
run: |
set -euo pipefail
git fetch --no-tags origin \
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
base="origin/${{ github.base_ref }}"
# Three dots: merge-base scoped, so commits landing on master mid-run
# are not counted as this PR's.
changed="$(git diff --name-only "$base"...HEAD)"
has() { printf '%s\n' "$changed" | grep -qx "$1"; }
if has man/httrack.1 && ! has html/httrack.man.html; then
echo "::error::man/httrack.1 changed but html/httrack.man.html did not."
echo "Regenerate it with: make -C man regen-man-html (needs the full groff package)."
exit 1
fi
echo "man/html sync OK."

72
.github/workflows/codeql.yml vendored
View File

@@ -1,72 +0,0 @@
# CodeQL static analysis (C). The security-extended suite covers the classes
# this codebase actually fights: overflows, tainted-size allocs, format bugs.
name: CodeQL
on:
push:
branches: [master]
pull_request:
schedule:
# Weekly re-scan of master so new/updated queries land without a push.
- cron: "17 4 * * 1"
permissions:
contents: read
concurrency:
group: codeql-${{ github.ref }}
cancel-in-progress: true
jobs:
analyze:
name: analyze (c-cpp)
runs-on: ubuntu-24.04
permissions:
contents: read
# Upload findings to the repo's code-scanning dashboard.
security-events: write
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev
- uses: github/codeql-action/init@v4
with:
languages: c-cpp
build-mode: manual
queries: security-extended
# fopen's umask-controlled 0666 is intended for mirror/cache/log
# output; the one credential file (cookies.txt) is kept 0600 on Unix.
config: |
paths-ignore:
# Demo callback samples, not part of libhttrack.
- libtest
# ProxyTrack: a separate legacy binary with no auth surface; its
# recv/cache-parse code trips taint queries by design.
- src/proxy
query-filters:
- exclude:
id: cpp/world-writable-file-creation
# Models auth-bypass-by-spoofing; httrack has no auth surface, its +/- crawl filter is a mirror boundary, not a security one.
- exclude:
id: cpp/user-controlled-bypass
# Manual build: CodeQL traces the compiler, so build exactly what ships.
- name: Build
run: |
set -euo pipefail
autoreconf -fi
./configure
make -j"$(nproc)"
- uses: github/codeql-action/analyze@v4
with:
category: "/language:c-cpp"

162
.github/workflows/macos-release.yml vendored
View File

@@ -1,162 +0,0 @@
# Signed, notarized HTTrack.dmg (#901). Its own workflow because a fork's pull request
# never gets the Developer ID secrets, and each run spends two notarization submissions.
name: macOS release
on:
push:
tags: ["[0-9]*"]
workflow_dispatch:
permissions:
contents: read
jobs:
dmg:
name: signed DMG (macOS arm64)
runs-on: macos-15
permissions:
contents: write # attach the DMG to the release the tag belongs to
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
# Notarization is minutes in; a missing secret should not surface there.
- name: Check the signing secrets are present
env:
MACOS_CERT_P12: ${{ secrets.MACOS_CERT_P12 }}
MACOS_CERT_PASSWORD: ${{ secrets.MACOS_CERT_PASSWORD }}
MACOS_NOTARY_KEY: ${{ secrets.MACOS_NOTARY_KEY }}
MACOS_NOTARY_KEY_ID: ${{ secrets.MACOS_NOTARY_KEY_ID }}
MACOS_NOTARY_ISSUER_ID: ${{ secrets.MACOS_NOTARY_ISSUER_ID }}
run: |
set -euo pipefail
hint="see secrets/apple/README.md in httrack-works"
: "${MACOS_CERT_P12:?unset repository secret ($hint)}"
: "${MACOS_CERT_PASSWORD:?unset repository secret ($hint)}"
: "${MACOS_NOTARY_KEY:?unset repository secret ($hint)}"
: "${MACOS_NOTARY_KEY_ID:?unset repository secret ($hint)}"
: "${MACOS_NOTARY_ISSUER_ID:?unset repository secret ($hint)}"
- name: Install build dependencies
run: |
set -euo pipefail
brew install autoconf automake libtool autoconf-archive brotli zstd
# --disable-shared keeps libhttrack inside the binaries.
- name: Build and install into a temp prefix
run: |
set -euo pipefail
ssl="$(brew --prefix openssl@3)"
brewp="$(brew --prefix)"
./bootstrap
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
LDFLAGS="-L${ssl}/lib -L${brewp}/lib" \
--prefix="$RUNNER_TEMP/inst" --disable-shared
make -j"$(sysctl -n hw.ncpu)"
make install
- name: Assemble HTTrack.app
run: |
set -euo pipefail
make macos-app APP_FLAGS="--prefix $RUNNER_TEMP/inst --out $RUNNER_TEMP"
plutil -lint "$RUNNER_TEMP/HTTrack.app/Contents/Info.plist"
- name: Import the Developer ID certificate
env:
CERT_P12: ${{ secrets.MACOS_CERT_P12 }}
CERT_PASSWORD: ${{ secrets.MACOS_CERT_PASSWORD }}
run: |
set -euo pipefail
kc="$RUNNER_TEMP/signing.keychain-db"
kcpass="$(openssl rand -base64 24)"
echo "::add-mask::$kcpass"
security create-keychain -p "$kcpass" "$kc"
# Notarization holds the job well past the five-minute default lock.
security set-keychain-settings -lut 21600 "$kc"
security unlock-keychain -p "$kcpass" "$kc"
# Armed before the file exists: a failed import must not leave the key on a
# disk that later steps run repository code against.
trap 'rm -f "$RUNNER_TEMP/cert.p12"' EXIT HUP INT TERM
# openssl, not base64: macOS spells the decode flag -D and GNU spells it -d.
printf '%s' "$CERT_P12" | openssl base64 -d -A -out "$RUNNER_TEMP/cert.p12"
security import "$RUNNER_TEMP/cert.p12" -k "$kc" -P "$CERT_PASSWORD" \
-T /usr/bin/codesign
# Without this codesign blocks on a UI prompt no runner can answer.
security set-key-partition-list -S apple-tool:,apple:,codesign: \
-s -k "$kcpass" "$kc" >/dev/null
security list-keychains -d user -s "$kc"
id="$(security find-identity -v -p codesigning "$kc" |
awk '/Developer ID Application/ {print $2; exit}')"
test -n "$id" || {
security find-identity -v -p codesigning "$kc" >&2
echo "the .p12 holds no Developer ID Application identity" >&2
exit 1
}
echo "SIGN_IDENTITY=$id" >>"$GITHUB_ENV"
- name: Sign, notarize and pack the DMG
env:
NOTARY_KEY: ${{ secrets.MACOS_NOTARY_KEY }}
NOTARY_KEY_ID: ${{ secrets.MACOS_NOTARY_KEY_ID }}
NOTARY_ISSUER_ID: ${{ secrets.MACOS_NOTARY_ISSUER_ID }}
run: |
set -euo pipefail
key="$RUNNER_TEMP/notary.p8"
trap 'rm -f "$key"' EXIT HUP INT TERM
printf '%s' "$NOTARY_KEY" | openssl base64 -d -A -out "$key"
bash tools/macos-release.sh --app "$RUNNER_TEMP/HTTrack.app" \
--identity "$SIGN_IDENTITY" --out "$RUNNER_TEMP" \
--notary-key "$key" --notary-key-id "$NOTARY_KEY_ID" \
--notary-issuer "$NOTARY_ISSUER_ID"
# The DMG round-trip is the one thing that can strip the stapled ticket, so assess
# the copy that came out of it rather than the app the signing step still holds.
- name: Assess a quarantined copy out of the DMG
run: |
set -euo pipefail
dmg="$(echo "$RUNNER_TEMP"/HTTrack-*.dmg)"
test -f "$dmg"
hdiutil attach -nobrowse -readonly -mountpoint "$RUNNER_TEMP/mnt" "$dmg"
trap 'hdiutil detach "$RUNNER_TEMP/mnt" >/dev/null || true' EXIT
app="$RUNNER_TEMP/quarantined/HTTrack.app"
ditto "$RUNNER_TEMP/mnt/HTTrack.app" "$app"
# On the root only, as a real download lands it; -r would also chase symlinks.
xattr -w com.apple.quarantine "0083;00000000;Safari;" "$app"
# --no-cache, or the verdict already cached for this cdhash answers instead.
log="$RUNNER_TEMP/assess.log"
spctl --assess --type exec --ignore-cache --no-cache -vv "$app" >"$log" 2>&1 || {
cat "$log" >&2
exit 1
}
cat "$log"
grep -q "source=Notarized Developer ID" "$log"
# spctl alone can be answered online; this is the offline-launch proof.
xcrun stapler validate "$app"
# Not a Gatekeeper check: the stub is a script, so nothing here goes through
# LaunchServices. It proves the hardened runtime did not break the dylib loads.
bash tests/webhttrack-smoke.sh "$app/Contents/Resources" "$app/Contents/MacOS/HTTrack"
- name: Upload the DMG
uses: actions/upload-artifact@v7
with:
name: HTTrack-macos-arm64-dmg
path: ${{ runner.temp }}/HTTrack-*.dmg
if-no-files-found: error
# Releases are cut by hand, so a tag can land before one exists.
- name: Attach the DMG to the release
if: github.ref_type == 'tag'
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
if gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
gh release upload "$GITHUB_REF_NAME" "$RUNNER_TEMP"/HTTrack-*.dmg --clobber
else
echo "no release $GITHUB_REF_NAME yet -- the DMG is on this run's artifacts"
fi
- name: Delete the signing keychain
if: always()
run: security delete-keychain "$RUNNER_TEMP/signing.keychain-db" || true

72
.github/workflows/screenshots.yml vendored
View File

@@ -1,72 +0,0 @@
# Documentation screenshots: drive the WebHTTrack UI and capture every screen.
# Manual, and deliberately not part of the build gate -- run it after adding or
# changing a UI page, and the artifact is the new set of shots.
name: screenshots
on:
# A push trigger as well as the button: workflow_dispatch is only offered for
# workflows that already exist on the default branch.
push:
paths:
- '.github/workflows/screenshots.yml'
- 'tools/screenshot-walk.py'
workflow_dispatch:
inputs:
lang:
description: 'lang.indexes number to render the UI in (1 = English)'
default: '1'
scale:
description: 'device pixel ratio'
default: '2'
permissions:
contents: read
jobs:
capture:
runs-on: ubuntu-24.04
timeout-minutes: 20
steps:
- uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
- name: Build
run: |
set -euo pipefail
autoreconf -fi
./configure
make -j"$(nproc)" -C src htsserver
- name: Install the browser
run: |
set -euo pipefail
pip install --quiet playwright
playwright install --with-deps chromium
- name: Capture
env:
LANGN: ${{ inputs.lang || '1' }}
SCALE: ${{ inputs.scale || '2' }}
run: |
set -euo pipefail
# --root is the dist root; the source tree already holds every piece
# htsserver reads, so this needs no install.
python3 tools/screenshot-walk.py --htsserver src/htsserver --root . \
--lang "$LANGN" --scale "$SCALE" --out shots
- name: Upload the screens
if: always()
uses: actions/upload-artifact@v7
with:
name: screenshots
path: shots
if-no-files-found: error

192
.github/workflows/windows-build.yml vendored
View File

@@ -1,192 +0,0 @@
# Windows build of the engine (VS2022 v143 + vcpkg): libhttrack, the CLI,
# webhttrack, proxytrack.
#
# The autotools CI covers the unix builds; this covers the MSVC one, which had
# no coverage at all and had drifted (the old .vcproj pinned OpenSSL 1.0.1j).
# libhttrack.dll is what the WinHTTrack GUI links against.
name: windows-build
on:
push:
branches: [master]
pull_request:
workflow_dispatch:
permissions:
contents: read
# Cancel superseded runs on the same branch or PR.
concurrency:
group: windows-build-${{ github.ref }}
cancel-in-progress: true
jobs:
libhttrack:
runs-on: windows-2022
strategy:
fail-fast: false
matrix:
platform: [x64, Win32]
configuration: [Release]
# Redirect vcpkg's default `files` binary cache into the workspace so
# actions/cache can persist it. vcpkg builds openssl/brotli/zlib/zstd from
# source otherwise, several minutes every run.
env:
VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\vcpkg_cache
steps:
- uses: actions/checkout@v7
with:
submodules: recursive # coucal lives in src/coucal
# Located through vswhere rather than microsoft/setup-msbuild: the repo
# only allows GitHub-owned actions.
- name: Find MSBuild
shell: pwsh
run: |
$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
$msbuild = & $vswhere -latest -products * -requires Microsoft.Component.MSBuild `
-find MSBuild\**\Bin\MSBuild.exe | Select-Object -First 1
if (-not $msbuild) { throw "MSBuild not found" }
Write-Host "MSBuild: $msbuild"
"MSBUILD=$msbuild" | Out-File -FilePath $env:GITHUB_ENV -Append
- name: Enable vcpkg MSBuild integration
shell: pwsh
run: vcpkg integrate install
# vcpkg errors if VCPKG_DEFAULT_BINARY_CACHE points at a missing dir, and
# actions/cache does not create it on a miss.
- name: Create the vcpkg binary cache directory
shell: pwsh
run: New-Item -ItemType Directory -Force -Path $env:VCPKG_DEFAULT_BINARY_CACHE | Out-Null
# x-gha is gone (vcpkg-tool #1662 dropped it after GitHub changed the cache
# API), so cache the binary archives directly. Keyed on the manifest, which
# carries the builtin-baseline, so a Dependabot bump busts it; restore-keys
# still seeds the unchanged ports' archives, so only the bumped one rebuilds.
- name: Cache vcpkg binary archives
uses: actions/cache@v6
with:
path: ${{ github.workspace }}\vcpkg_cache
key: vcpkg-${{ matrix.platform }}-${{ hashFiles('src/vcpkg.json') }}
restore-keys: |
vcpkg-${{ matrix.platform }}-
# The runner image's vcpkg checkout is pinned to some commit; our manifest's
# builtin-baseline is usually newer, so `git show <baseline>:versions/...`
# fails until that commit is local. Fetch exactly the pinned baseline (read
# from the manifest, so Dependabot bumps need no workflow edit).
- name: Fetch the pinned vcpkg baseline
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$root = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { "C:\vcpkg" }
$baseline = (Get-Content src/vcpkg.json -Raw | ConvertFrom-Json).'builtin-baseline'
if (-not $baseline) { throw "no builtin-baseline in src/vcpkg.json" }
git -C $root fetch --no-tags origin $baseline
if ($LASTEXITCODE -ne 0) { throw "could not fetch vcpkg baseline $baseline" }
# httrack and webhttrack carry a ProjectReference to libhttrack, so building
# them builds it first; proxytrack is standalone.
- name: Build
shell: pwsh
run: |
foreach ($proj in @("libhttrack", "httrack", "webhttrack", "proxytrack")) {
& $env:MSBUILD "src\$proj.vcxproj" `
/m `
/p:Configuration=${{ matrix.configuration }} `
/p:Platform=${{ matrix.platform }} `
/p:VcpkgEnableManifest=true `
/flp:LogFile=msbuild-$proj.log`;Verbosity=normal
if ($LASTEXITCODE -ne 0) { throw "$proj failed" }
}
# WinHTTrack links src\$(Platform)\$(Configuration)\libhttrack.lib, so the
# import lib and the DLL both have to land there under that exact name.
- name: Check the binaries landed where they are expected
shell: pwsh
run: |
$dir = "src\${{ matrix.platform }}\${{ matrix.configuration }}"
$want = @("libhttrack.dll", "libhttrack.lib", "httrack.exe",
"webhttrack.exe", "proxytrack.exe")
foreach ($f in $want) {
if (-not (Test-Path "$dir\$f")) { throw "missing $dir\$f" }
Write-Host "found $dir\$f"
}
# A CRT split across the exe/DLL boundary links fine and crashes on the
# first cross-heap free, so assert both sides import the *same* vcruntime.
# Also assert the machine type: Test-Path alone would accept an x64 binary
# sitting in the Win32 output dir.
- name: Check the CRT is shared across the boundary
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$dir = "src\${{ matrix.platform }}\${{ matrix.configuration }}"
$dumpbin = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" `
-latest -find VC\Tools\MSVC\**\bin\Host*\*\dumpbin.exe | Select-Object -First 1
if (-not $dumpbin) { throw "dumpbin not found" }
$wantMachine = if ("${{ matrix.platform }}" -eq "x64") { "x64" } else { "x86" }
function Get-Crt($path) {
$out = & $dumpbin /dependents /headers $path
if ($LASTEXITCODE -ne 0) { throw "dumpbin failed on $path" }
,@($out | Select-String -Pattern '^\s*(vcruntime\d+d?\.dll)$' |
ForEach-Object { $_.Matches[0].Groups[1].Value.ToLower() })
}
# Negative control: cmd.exe links the old msvcrt, not vcruntime140. If
# this came back non-empty the detector would match anything, and the
# assertion below would pass on a mismatched CRT too.
if ((Get-Crt "$env:SystemRoot\System32\cmd.exe").Count -ne 0) {
throw "negative control failed: the vcruntime detector matches anything"
}
$crts = @{}
foreach ($f in @("libhttrack.dll", "httrack.exe")) {
$crt = Get-Crt "$dir\$f"
if ($crt.Count -eq 0) { throw "$f imports no vcruntime: static or mismatched CRT" }
$crts[$f] = $crt
$hdr = & $dumpbin /headers "$dir\$f"
if (-not ($hdr | Select-String -SimpleMatch "machine ($wantMachine)")) {
throw "$f is not $wantMachine"
}
Write-Host "$f -> $($crt -join ',') ($wantMachine)"
}
$a = $crts["libhttrack.dll"] | Sort-Object
$b = $crts["httrack.exe"] | Sort-Object
if (Compare-Object $a $b) {
throw "CRT mismatch across the boundary: libhttrack.dll=$a httrack.exe=$b"
}
# The engine ships ~90 tests and none had ever run on Windows: "make check"
# is Linux/macOS only. These are the offline ones, driven from Git Bash
# against the native httrack.exe. They subsume the self-tests this step
# used to run inline (codecs, cache, fsize).
# The *_local-* ones crawl the bundled Python server over loopback: the real
# TLS handshake, cache and file writer, which nothing else on Windows covers.
- name: Run the engine test suite (offline tests)
shell: bash
working-directory: tests
timeout-minutes: 45
run: |
bash ./ci-windows-suite.sh \
"$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"
- name: Upload the test logs
if: always()
uses: actions/upload-artifact@v7
with:
name: engine-tests-${{ matrix.platform }}-${{ matrix.configuration }}
path: tests/*.log
if-no-files-found: warn
- name: Upload MSBuild logs
if: always()
uses: actions/upload-artifact@v7
with:
name: msbuild-${{ matrix.platform }}-${{ matrix.configuration }}
path: msbuild-*.log
if-no-files-found: ignore

21
.gitignore vendored
View File

@@ -24,11 +24,8 @@ Makefile.in
/config.log
/config.status
/stamp-h1
# src/webhttrack.in is the source; an in-tree build generates this one (#887).
/src/webhttrack
Makefile
.deps/
.dirstamp
.libs/
*.o
*.lo
@@ -37,24 +34,8 @@ Makefile
*.so.*
*.a
# Built into the checkout by an in-tree "make" / "make check".
/src/httrack
/src/htsserver
/src/proxytrack
/tools/Info.plist
/tests/stringoom
/tests/*.log
/tests/*.trs
# make dist output; dist/ holds httrack-gh-release staging artifacts.
# make dist output.
/httrack-*.tar.gz
/dist/
# Editor / autotools backup files.
*~
# Python bytecode (tests/local-server.py).
__pycache__/
# Per-checkout Claude Code rules (symlink into a local sandbox).
/CLAUDE.local.md

83
AGENTS.md
View File

@@ -6,38 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
## Build & test
- Fresh clone first: `git submodule update --init src/coucal`
- `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
automake, libtool), then `bash configure && make -j"$(nproc)" && make check
-j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
automake's parallel harness and each crawl test binds its own ephemeral-port
server, so `-j` never contends and a multi-minute serial run drops to
seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
automatically; only a test slower than the current longest raises the floor.
On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
spend much of their wall time asleep (server trickles, httrack self-pacing),
so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
platform, macOS included: the test server raises its listen backlog
(`request_queue_size`) so macOS/BSD don't drop connections under a parallel
`-c16` bigcrawl the way Python's default backlog of 5 did.
Or run `sh build.sh` to do bootstrap + configure + make in one shot.
- A `tests/NN_*.test` runs only if listed in `tests/tests-list.mk`; an
unregistered file is silently skipped.
- `make check` prepends the build's `src/` to `PATH`, but a hand-run `.test` does
not — an installed `/usr/bin/httrack` then shadows your build. Run via `make
check`, or `PATH="<bld>/src:$PATH"` for a manual run.
- Give new `.test` scripts `set -e`: the older ones predate the rule, so several
`local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
- Run teardown with errexit off: `trap 'set +e; cleanup' EXIT`. Under `set -e` a
failing cleanup command becomes the test's exit status (#773). Keep the other
signals on their own `trap` line, or errexit stays off for the rest of the run.
The guard also resets `$?`, so save it first if teardown reads it.
- Never pipe into `grep -q`: it exits on the first match, so whatever the
producer had left to write takes SIGPIPE, and under `pipefail` that becomes
the pipeline's status. `cmd | grep -q M && fail` then never fires and a probe
that proved nothing reads as "marker absent"; `cmd | grep -q M || fail` fails
a test whose marker was present. bash issues one `write()` per line, so any
match that is not on the last line is exposed. Capture the reply, assert the
status line it must carry (an empty, truncated or redirected one is
marker-free too), then match with a here-string: `grep -q M <<<"$reply"`.
automake, libtool), then `bash configure && make && make check`. Or run
`sh build.sh` to do bootstrap + configure + make in one shot.
## Hard invariants
- **Generated autotools files are NOT in git.** `configure`, every
@@ -50,41 +20,14 @@ the operational checklist: toolchain, invariants, and how to ship a change.
- **Format only changed lines** with `git clang-format` (clang-format 19). Never
reformat untouched code: the engine was formatted by an old tool and won't
round-trip.
- **Byte-safe edits.** A few tracked files carry raw ISO-8859-1 high bytes
(French comments): `src/htsconcat.c`, `lang/*.txt`, `html/contact.html`, and
the `fuzz/corpus/*` vectors. Edit those byte-wise (`perl -0pi`, `sed`), not
through a tool that re-encodes to UTF-8 and corrupts them. The rest of the tree
is UTF-8 and safe to edit normally.
- **Byte-safe edits.** Files with raw high bytes are ISO-8859-1 (French
comments). Edit them byte-wise (`perl -0pi`, `sed`), not through a tool that
re-encodes to UTF-8 and corrupts them.
## Security (HTTrack parses hostile input off the network)
- Bounds-check every copy. Overflow-safe form: put the untrusted value alone,
`untrusted < limit - controlled` — never `controlled + untrusted < limit`,
which can wrap and pass.
- **Abort or clip is a decision, not a default.** The `*_safe_` helpers
(`strcpybuff`, `strlcpybuff`, `strcatbuff`) **abort** on overflow. Right for
our own data, wrong for anything read back from a cache, a header or the
wire, where it trades a memory smash for a crash on malformed input. Clip
with `dst[0] = '\0'; strlncatbuff(dst, src, size, size - 1)`.
- **A warning class is not the unsafe set.** `-Wformat-truncation` fires only on
a *bounded* `snprintf` whose return is discarded, so an unbounded `sprintf`
into the same buffer never appears on it. Before scoping a hardening pass off
compiler output, grep the unguarded forms yourself (`\bsprintf\s*\(`,
`\bstrcpy\s*\(`, `\bstrcat\s*\(`).
## C conventions
- **Use the `*t` allocator wrappers, never raw libc** (`htssafe.h`):
`malloct`/`calloct`/`realloct`/`freet`/`strdupt`, in test and selftest code
too. `freet` NULLs its (lvalue) argument and tolerates NULL; `calloct(n, sz)`
keeps calloc's arg order. Only exception: storing or calling a libc symbol
itself (e.g. a resolver-backend function pointer).
- **Exported API is `HTSEXT_API`.** Everything else is hidden by
`-fvisibility=hidden` and free to change (check with `nm -D --defined-only
libhttrack.so`). Touching an installed-header struct (see `DevIncludes_DATA` in
`src/Makefile.am`) or an exported signature is an ABI break — flag and discuss,
bump the soname, and prefer keeping the old entry point beside a new one.
- **Windows ABI is free to break, POSIX is not.** The Windows DLL ships next to
the exe with no soname contract, so a `_WIN32`-only ABI change needs no
deprecation dance; POSIX/ELF keeps the flag-discuss-bump rules.
## Code & prose
- Be terse. Comment the why, in English; translate French comments you touch.
@@ -106,17 +49,6 @@ Before pushing, and when reviewing others, don't skim for bugs:
layout/ABI, cache/wire format, or a security path? A static or unit check
isn't enough; exercise the wrong behavior at runtime. Claude Code:
`/review-recipe`.
- **Poison a canary, never compare it against zero.** Checking that a
neighbouring field is still `'\0'` cannot see the stray NUL an off-by-one
terminator writes — the exact bug the canary is there for. Fill it with a
non-zero byte, and prove it by killing both the stray-`'X'` and the
stray-NUL mutant. Neither ASan nor `_FORTIFY_SOURCE` sees an overflow that
lands inside the same struct.
- **Overshoot every destination, not one.** A bounds test that oversizes a
single field cannot tell a per-field bound from a one-size-fits-all one, nor
from a fix that bounds that field and leaves its neighbours raw. Exercise
each destination the path touches, spanning at least two capacities, and
check what the code actually emits before writing the expected values.
## Commits
- **Sign-off is mandatory.** Every commit carries a `Signed-off-by` trailer:
@@ -124,9 +56,8 @@ Before pushing, and when reviewing others, don't skim for bugs:
- **Co-Authored-By is mandatory for AI-assisted commits.** Carry a
`Co-Authored-By:` trailer naming the assistant. Attribute there, never in a
PR-body footer.
- PRs are squash-merged: one commit per PR lands on master, built from the PR
title and description, so those are what the history keeps. The branch's
intermediate commits are not preserved.
- PRs land as a merge commit; every commit on the branch goes onto master, so
keep each commit message clean and meaningful.
## PR descriptions
- Plain concise prose; lead with what changed and why. No What/Why/How template.

4
CONTRIBUTING.md
View File

@@ -6,8 +6,8 @@ with an AI assistant? The operational checklist is [AGENTS.md](AGENTS.md).
## Pull requests
- One change per PR. Small diffs merge fast.
- PRs are squash-merged: one commit per PR goes onto master, built from the PR
title and description, so those are the history. Explain *why* there.
- PRs land as a merge commit, so the branch's commits go onto master as-is: keep
each commit message clean and explain *why*.
- Be terse in the PR title and description: name the problem, not the fix, don't
restate the diff, and calibrate length to the change.
- Add or update tests for engine changes (`tests/`), and keep CI green.

26
Makefile.am
View File

@@ -1,38 +1,18 @@
SUBDIRS = src man m4 libtest templates lang html tests fuzz
SUBDIRS = src man m4 libtest templates lang html tests
ACLOCAL_AMFLAGS = -I m4
EXTRA_DIST = INSTALL.Linux \
gpl-fr.txt license.txt greetings.txt history.txt \
httrack-doc.html lang.def README.md tools/mkdeb.sh \
tools/macos-app.sh tools/macos-bundle.sh tools/macos-release.sh \
tools/httrack-launcher.c \
tools/Info.plist.in tools/HTTrack.icns \
bootstrap build.sh
# Build the signed Debian packages from a clean source export. Pass the signing
# key and other options through DEB_FLAGS, e.g.:
# make deb DEB_FLAGS="--key BB71C7E6CB1AD8FAF53FE42A60C3AA7180598EFB"
# See tools/mkdeb.sh --help for all options. Not $(SHELL): mkdeb.sh is bash, and dash chokes (#891).
# See tools/mkdeb.sh --help for all options.
DEB_FLAGS =
deb:
$(BASH_SHELL) $(top_srcdir)/tools/mkdeb.sh $(DEB_FLAGS)
$(SHELL) $(top_srcdir)/tools/mkdeb.sh $(DEB_FLAGS)
.PHONY: deb
# Assemble the macOS application bundle from an installed prefix, e.g.
# make macos-app APP_FLAGS="--prefix /tmp/inst --out /tmp"
APP_FLAGS =
macos-app:
$(SHELL) $(top_srcdir)/tools/macos-app.sh --plist tools/Info.plist \
--icon $(top_srcdir)/tools/HTTrack.icns $(APP_FLAGS)
.PHONY: macos-app
# Sign, notarize and pack that bundle into the DMG we publish, e.g.
# make macos-release RELEASE_FLAGS="--app /tmp/HTTrack.app --identity ABCD1234 \
# --notary-key AuthKey.p8 --notary-key-id KEYID --notary-issuer ISSUERID"
# See tools/macos-release.sh for the rest; .github/workflows/macos-release.yml runs it.
RELEASE_FLAGS =
macos-release:
$(SHELL) $(top_srcdir)/tools/macos-release.sh $(RELEASE_FLAGS)
.PHONY: macos-release

1
README
View File

@@ -40,6 +40,7 @@ These are the principals limits of HTTrack for that moment. Note that we did not
that would have solved them.
- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
- Some java classes may not find some files on them (class included)
- Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*

186
configure.ac
View File

@@ -1,6 +1,6 @@
AC_PREREQ([2.71])
AC_INIT([httrack], [3.49.15], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
AC_INIT([httrack], [3.49.10], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
AC_COPYRIGHT([
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,79 +29,13 @@ AC_CONFIG_SRCDIR(src/httrack.c)
AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_HEADERS(config.h)
AM_INIT_AUTOMAKE([subdir-objects])
# 3:7:0: htsblk gained a tail field and lien_back embeds it by value, so
# lien_back.is_update and everything after it shift +8 (httrackp's own tail growth
# moves nothing). Soname stays .so.3: HTTrackQt is the only consumer of the installed
# headers, so a libhttrack4 rename isn't worth it.
# 3:2:0: 3.49.10 only appends tail fields to the options struct (no existing
# symbol or offset changed vs 3.49.9), so it stays soname .so.3; bump revision.
# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
VERSION_INFO="3:7:0"
VERSION_INFO="3:2:0"
AM_MAINTAINER_MODE
AC_USE_SYSTEM_EXTENSIONS
# A real bash, for "make deb" and the test harness. Not searched into BASH: bash presets
# that to its own path, and macOS /bin/sh is a bash, so the macro never searches (#895).
# BASH_SHELL isn't preset the way BASH is, so AC_ARG_VAR needs no guard. Kept ahead of the
# compiler probes so a bad override dies before them.
AC_ARG_VAR([BASH_SHELL], [path to a real (non-POSIX-mode) bash])
# AC_PATH_PROGS drops a relative override and searches instead, which loses the user's intent.
# Nothing quotes $(BASH_SHELL) in the Makefiles, and quoting could not save it anyway: make
# splits on whitespace, expands '$' and treats '#' as a comment before any shell sees it.
case $BASH_SHELL in
*[[[:space:]]]* | *'#'* | *'$'* | *'`'* | *'\'* | *'"'* | *"'"* | *';'* | *'&'* | *'|'* | \
*'<'* | *'>'* | *'('* | *')'* | *'*'* | *'?'* | *'@<:@'* | *'@:>@'* | *'{'* | *'}'*)
AC_MSG_ERROR([BASH_SHELL must not contain shell or make metacharacters, got: $BASH_SHELL]) ;;
'' | [[\\/]]* | ?:[[\\/]]*) ;;
*) AC_MSG_ERROR([BASH_SHELL must be an absolute path, got: $BASH_SHELL]) ;;
esac
hts_bash_override=$BASH_SHELL
AC_PATH_PROGS([BASH_SHELL], [bash], [/bin/bash])
# An absolute override is taken verbatim, so BASH_SHELL=/bin/sh would put #895 back and only
# surface at "make check" or "make deb" (#908). What we found ourselves is only a warning:
# a box with no bash still builds, it just cannot run those two.
AC_MSG_CHECKING([whether $BASH_SHELL is a bash outside POSIX mode])
hts_bash_why=
hts_bash_env=no
# AS_EXECUTABLE_P, not "test -x": the PATH search above already demands a regular file, and
# bash blocks forever reading a FIFO it failed to exec, so -x alone hangs configure (#922).
if ! AS_EXECUTABLE_P(["$BASH_SHELL"]); then
hts_bash_why="not an executable regular file"
elif test -z "$("$BASH_SHELL" -c 'echo "${BASH_VERSINFO[[0]]}"' 2>/dev/null)"; then
# Not BASH_VERSION: that is an ordinary variable, so any shell echoes back a spoofed one.
hts_bash_why="not a bash: it reports no BASH_VERSINFO"
else
# sh-mode bash reports a version too, so only SHELLOPTS tells the two apart.
case $("$BASH_SHELL" -c 'echo ":$SHELLOPTS:"' 2>/dev/null) in
*:posix:*)
hts_bash_why="a bash in POSIX sh-mode"
# POSIXLY_CORRECT and an exported SHELLOPTS do that to every bash on the box, so no path
# can pass and blaming this one would send the user hunting for another. Reading them
# here would not do: configure puts its own shell in posix mode, which sets both.
case $(env -u POSIXLY_CORRECT -u SHELLOPTS "$BASH_SHELL" -c 'echo ":$SHELLOPTS:"' 2>/dev/null) in
'' | *:posix:*) ;; # no "env -u", or posix whatever the environment: blame the path
*) hts_bash_env=yes ;;
esac
;;
esac
fi
if test -z "$hts_bash_why"; then
AC_MSG_RESULT([yes])
else
AC_MSG_RESULT([no])
hts_bash_msg="POSIXLY_CORRECT or SHELLOPTS forces every bash into POSIX sh-mode, $BASH_SHELL included. Clear them for configure and for make, which hands them to make check and make deb: env -u POSIXLY_CORRECT -u SHELLOPTS ..."
if test "$hts_bash_env" = yes; then
if test -n "$hts_bash_override"; then
AC_MSG_ERROR([$hts_bash_msg])
fi
AC_MSG_WARN([$hts_bash_msg])
else
if test -n "$hts_bash_override"; then
AC_MSG_ERROR([BASH_SHELL=$BASH_SHELL is $hts_bash_why])
fi
AC_MSG_WARN([no usable bash found: $BASH_SHELL is $hts_bash_why. "make check" and "make deb" need one; pass BASH_SHELL=/path/to/bash])
fi
fi
AC_PROG_CC
AM_PROG_CC_C_O
m4_warn([obsolete],
@@ -112,13 +46,14 @@ m4_warn([obsolete],
# script's behavior did not change. They are probably safe to remove.
AC_CHECK_INCLUDES_DEFAULT
AC_PROG_EGREP
# $(SED) substitutes $(datadir) into src/webhttrack
AC_PROG_SED
LT_INIT
AC_PROG_LN_S
LT_INIT
# bash, used to run the test scripts (see tests/Makefile.am TEST_LOG_COMPILER)
AC_PATH_PROGS([BASH], [bash], [/bin/bash])
# Export LD_LIBRARY_PATH name or equivalent.
AC_SUBST(SHLIBPATH_VAR,$shlibpath_var)
@@ -129,11 +64,10 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
AC_SUBST(VERSION_INFO)
### Default CFLAGS
# No -Wdeclaration-after-statement: nothing sets -std=, so this builds as gnu17.
DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
-Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
-Wstrict-prototypes -Wmissing-prototypes \
-Wmissing-declarations \
-Wmissing-declarations -Wdeclaration-after-statement \
-Wpointer-arith -Wsequence-point -Wnested-externs \
-D_REENTRANT"
AC_SUBST(DEFAULT_CFLAGS)
@@ -141,47 +75,33 @@ DEFAULT_LDFLAGS=""
AC_SUBST(DEFAULT_LDFLAGS)
### Additional flags (if supported)
# -Werror on probes: exit-status-only checks let clang's warn-on-unknown-flag through.
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"], [], [-Werror])
# -Wformat-nonliteral needs -Wformat in the probe or gcc rejects it as ignored.
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"], [], [-Werror -Wformat])
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"])
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"])
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"])
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"])
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"])
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"])
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"])
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"])
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"])
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"])
# Make htssafe.h's pointer-dest 'warning' attribute a hard error in our build
# (migration is at zero; a new char* dest is a regression). gcc/clang each take
# only their own spelling; downstream keeps the plain warning, not a build break.
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"], [], [-Werror])], [-Werror])
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"], [], [-Werror])
# backtrace() unwinds through these; armhf, unlike amd64/arm64, defaults them off
# and printed frameless crash reports.
AX_CHECK_COMPILE_FLAG([-fasynchronous-unwind-tables], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fasynchronous-unwind-tables"], [], [-Werror])
# No --discard-all: it drops the local symbols naming every static function, so
# a trace misattributes them to the nearest surviving global. Costs 0.6% size.
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--no-undefined"])
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
# sanitizer builds, whose interceptors want the unfortified calls.
case "$CFLAGS" in
*-fsanitize=*) ;;
*) AX_ADD_FORTIFY_SOURCE ;;
esac
# Force libc back into DT_NEEDED for libraries that reach it only through
# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
# ld rejects these flags and links libSystem unconditionally.
# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
# linker; Apple ld rejects these flags and links libSystem unconditionally.
AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
AC_SUBST([LIBC_FORCE_LINK])
@@ -189,18 +109,13 @@ AC_SUBST([LIBC_FORCE_LINK])
### PIE
CFLAGS_PIE=""
LDFLAGS_PIE=""
AX_CHECK_COMPILE_FLAG([-fpie], [CFLAGS_PIE="-fpie"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-fpie -pie], [CFLAGS_PIE="-fpie -pie"])
AX_CHECK_LINK_FLAG([-pie], [LDFLAGS_PIE="-pie"])
AC_SUBST([CFLAGS_PIE])
AC_SUBST([LDFLAGS_PIE])
# Ties a crash trace from a stripped build back to its separate debug symbols.
AX_CHECK_LINK_FLAG([-Wl,--build-id], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--build-id"])
# tools/macos-app.sh rewrites load commands to @rpath, which needs header room ld64 does
# not leave by default; the probe fails on GNU ld, so this self-gates on Darwin.
AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names],
[DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-headerpad_max_install_names"])
## Export all symbols for backtraces
AX_CHECK_COMPILE_FLAG([-rdynamic], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -rdynamic"])
### Check for -fvisibility=hidden support
gl_VISIBILITY
@@ -244,15 +159,11 @@ AC_CHECK_TYPE(sa_family_t, [], [AC_DEFINE([sa_family_t], [uint16_t], [sa_family_
AX_CHECK_ALIGNED_ACCESS_REQUIRED
# check for various headers
AC_CHECK_HEADERS([execinfo.h sys/ioctl.h])
AC_CHECK_HEADERS([execinfo.h])
### zlib (mandatory)
### zlib
CHECK_ZLIB()
### brotli and zstd content codings (optional)
CHECK_BROTLI()
CHECK_ZSTD()
### OpenSSL is explicitly enabled/disabled ?
AC_MSG_CHECKING(whether to enable https support)
AC_ARG_ENABLE([https],
@@ -388,37 +299,6 @@ AC_ARG_ENABLE([online-unit-tests],
])
AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
## libFuzzer harnesses (fuzz/); requires clang
AC_MSG_CHECKING(whether to build fuzzers)
AC_ARG_ENABLE([fuzzers],
[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
[
case "${enableval}" in
no|yes)
fuzzers=$enableval
AC_MSG_RESULT($enableval)
;;
*)
AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
;;
esac
],
[
fuzzers=no
AC_MSG_RESULT(no)
])
if test x"$fuzzers" = x"yes"; then
# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
if test x"$enable_shared" != x"no"; then
AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
fi
fi
AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
# Final output
AC_CONFIG_FILES([
Makefile
@@ -430,7 +310,5 @@ lang/Makefile
html/Makefile
libtest/Makefile
tests/Makefile
fuzz/Makefile
tools/Info.plist
])
AC_OUTPUT

55
debian/changelog vendored
View File

@@ -1,58 +1,3 @@
httrack (3.49.15-1) unstable; urgency=medium
* New upstream release: single-file mirrors with inlined assets, sitemap
ingestion, a change report between crawls, and numerous --update, FTP and
parser fixes; full list in history.txt.
* httrack-doc grows by roughly 1 MB: the offline guide now ships screenshots
of the WinHTTrack, WebHTTrack and Android interfaces.
-- Xavier Roche <xavier@debian.org> Thu, 30 Jul 2026 10:40:08 +0200
httrack (3.49.14-1) unstable; urgency=medium
* New upstream release: WARC/WACZ archive output, Windows long-path support,
named -%F footer fields, and encoding fixes for non-ASCII project paths;
full list in history.txt.
-- Xavier Roche <xavier@debian.org> Fri, 24 Jul 2026 08:01:43 +0200
httrack (3.49.13-1) unstable; urgency=medium
* New upstream release: SOCKS5 and CONNECT proxy support, brotli and zstd
content decoding, --update data-loss fixes, and hardening of several
network-facing parsers; full list in history.txt.
* Add libbrotli-dev and libzstd-dev to Build-Depends for the new content
codings.
* libhttrack3 and libhttrack-dev no longer ship libhtsjava.so.3*: the
obsolete Java-applet parser was removed upstream. It was a dlopen plugin
with no reverse dependencies, so this is a plain file drop on upgrade;
libhttrack.so.3 itself is unchanged and needs no rename.
* Drop the htsjava attribution from debian/copyright.
-- Xavier Roche <xavier@debian.org> Fri, 17 Jul 2026 10:49:52 +0200
httrack (3.49.12-1) unstable; urgency=medium
* New upstream release: security and crawl-correctness fixes (remote stack
overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
world-readable cookies.txt, filter-pattern denial of service) plus a new
--why filter diagnostic; full list in history.txt.
* Build with _FORTIFY_SOURCE and -fstack-protector-strong.
-- Xavier Roche <xavier@debian.org> Fri, 10 Jul 2026 21:11:09 +0200
httrack (3.49.11-1) unstable; urgency=medium
* New upstream release: crawl correctness and security fixes (network-facing
buffer overflows, file-type detection, redirect handling) and modernized
web defaults; full list in history.txt.
* Add DEP-12 upstream metadata (#466).
* Bump debhelper compat to 14 (#466).
* Drop the redundant Priority field and update the NMU lintian override to
the current tag names (#466).
-- Xavier Roche <xavier@debian.org> Sun, 05 Jul 2026 00:03:18 +0200
httrack (3.49.10-2) unstable; urgency=medium
* Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is

3
debian/control vendored
View File

@@ -1,8 +1,9 @@
Source: httrack
Section: web
Priority: optional
Maintainer: Xavier Roche <roche@httrack.com>
Standards-Version: 4.7.4
Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev, libbrotli-dev, libzstd-dev
Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
Rules-Requires-Root: no
Homepage: http://www.httrack.com
Vcs-Git: https://github.com/xroche/httrack.git

5
debian/copyright vendored
View File

@@ -7,8 +7,9 @@ Files: *
Copyright: 1998-2026 Xavier Roche and other contributors
License: GPL-3+
Comment:
htsbasenet.h links against the system OpenSSL library (originally by Eric
Young); no OpenSSL/SSLeay code is bundled here.
The engine includes contributions from Yann Philippot (src/htsjava.c,
src/htsjava.h). htsbasenet.h links against the system OpenSSL library
(originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
Files: src/minizip/*
Copyright: 1998-2010 Gilles Vollant

1
debian/libhttrack-dev.files vendored
View File

@@ -1,4 +1,5 @@
usr/include/httrack
usr/lib/*/libhttrack.{so}
usr/lib/*/libhtsjava.{so}
usr/share/httrack/libtest
usr/lib/*/httrack/libtest

1
debian/libhttrack3.files vendored
View File

@@ -1,2 +1,3 @@
usr/lib/*/libhttrack.so.3*
usr/lib/*/libhtsjava.so.3*
usr/share/httrack/templates

1
debian/patches/series vendored Normal file
View File

@@ -0,0 +1 @@
skip-local-pause-test-without-python3.patch

29
debian/patches/skip-local-pause-test-without-python3.patch vendored Normal file
View File

@@ -0,0 +1,29 @@
Description: skip 28_local-pause when python3 is absent (fixes FTBFS on buildds)
The local-server tests skip (exit 77) when python3 is missing, but
28_local-pause wrapped local-crawl.sh in a command substitution that swallowed
that skip. On the python3-less buildd chroot the test then ran serverless, both
crawls finished in 0s, and the 0s delta was reported as a failure. Guard the
test on python3 up front, like its siblings, so it skips cleanly.
Author: Xavier Roche <roche@httrack.com>
Origin: upstream, https://github.com/xroche/httrack/pull/445
Applied-Upstream: https://github.com/xroche/httrack/commit/45279d7
Forwarded: not-needed
Last-Update: 2026-06-28
diff --git a/tests/28_local-pause.test b/tests/28_local-pause.test
index 8505a75..add95d1 100755
--- a/tests/28_local-pause.test
+++ b/tests/28_local-pause.test
@@ -9,6 +9,13 @@ set -e
: "${top_srcdir:=..}"
+# python3 runs the local server (mirror local-crawl.sh); skip when absent, else
+# run() swallows its exit-77 and the serverless 0s/0s crawl looks like a fail.
+command -v python3 >/dev/null || {
+ echo "python3 not found; skipping local crawl tests"
+ exit 77
+}
+
run() { # echoes the wall-clock seconds of one crawl
local t0 t1
t0=$(date +%s)

4
debian/source/lintian-overrides vendored
View File

@@ -1,6 +1,4 @@
# Maintainer uploads sign the changelog as xavier@debian.org while the control
# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
httrack source: no-nmu-in-changelog
httrack source: changelog-should-mention-nmu
httrack source: source-nmu-has-incorrect-version-number
# The bundled HTML pages are the genuine upstream documentation taken from

6
debian/upstream/metadata vendored
View File

@@ -1,6 +0,0 @@
---
Repository: https://github.com/xroche/httrack.git
Repository-Browse: https://github.com/xroche/httrack
Bug-Database: https://github.com/xroche/httrack/issues
Bug-Submit: https://github.com/xroche/httrack/issues/new
Contact: Xavier Roche <roche@httrack.com>

65
fuzz/Makefile.am
View File

@@ -1,65 +0,0 @@
# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
if FUZZERS
noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx \
fuzz-htsparse fuzz-singlefile fuzz-sitemap fuzz-arc
endif
AM_CPPFLAGS = \
@DEFAULT_CFLAGS@ \
@THREADS_CFLAGS@ \
@V6_FLAG@ \
@LFS_FLAG@ \
-I$(top_srcdir)/src \
-I$(top_srcdir)/src/coucal
# Static-link libhttrack.la: the internal symbols are hidden in the .so.
AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
fuzz_url_SOURCES = fuzz-url.c fuzz.h
fuzz_header_SOURCES = fuzz-header.c fuzz.h
fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
fuzz_htsparse_SOURCES = fuzz-htsparse.c fuzz.h
fuzz_singlefile_SOURCES = fuzz-singlefile.c fuzz.h
fuzz_sitemap_SOURCES = fuzz-sitemap.c fuzz.h
# proxytrack does not link libhttrack, so its store compiles into the harness;
# coucal, minizip and md5 still come from the static libhttrack above.
fuzz_arc_SOURCES = fuzz-arc.c fuzz.h $(top_srcdir)/src/proxy/store.c
fuzz_arc_CPPFLAGS = $(AM_CPPFLAGS) -DZLIB_CONST
# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
EXTRA_DIST = README.md run-fuzzers.sh \
corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
corpus/idna/idna.txt corpus/idna/unicode.txt \
corpus/idna/regress-multilabel-leak.txt \
corpus/entities/entities.txt \
corpus/unescape/percent.txt \
corpus/filters/filter.bin corpus/filters/filter-size.bin \
corpus/filters/regress-empty-subject-unique.bin \
corpus/filters/redos-star-classes.bin \
corpus/filters/regress-classdepth-timeout.bin \
corpus/url/http-url.txt corpus/url/relative-path.txt \
corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
corpus/header/full-response.txt corpus/header/redirect.txt \
corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
corpus/cachendx/regress-overadvance.bin \
corpus/cachendx/regress-truncated-entry.bin \
corpus/htsparse/basic.html corpus/htsparse/script-inscript.html \
corpus/htsparse/meta-usemap.html corpus/htsparse/malformed.html \
corpus/singlefile/img-src.html corpus/singlefile/link-rel.html \
corpus/singlefile/style-block.html corpus/singlefile/style-attr.html \
corpus/singlefile/srcset.html corpus/singlefile/rawtext.html \
corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html \
corpus/sitemap/urlset.xml corpus/sitemap/sitemapindex.xml \
corpus/sitemap/truncated.xml corpus/sitemap/urlset.xml.gz \
corpus/arc/roundtrip.arc corpus/arc/truncated.arc \
corpus/arc/regress-null-body.arc

17
fuzz/README.md
View File

@@ -1,17 +0,0 @@
# Fuzzing httrack
libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
```sh
./bootstrap
mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
LDFLAGS="-fsanitize=address,undefined" \
bash /path/to/httrack/configure --enable-fuzzers --disable-shared
make
bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60 # 60s per target
```
Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.
`fuzz-arc` is the odd one out: it drives proxytrack's `.arc` reader the way `--convert` does, through a temp file rather than a buffer, and it compiles `src/proxy/store.c` into the harness because proxytrack does not link libhttrack. Both readers and the writer print to stderr on malformed input, so pass `-close_fd_mask=2` for anything longer than a corpus replay.

9
fuzz/corpus/arc/regress-null-body.arc
View File

@@ -1,9 +0,0 @@
filedesc://t.arc 0.0.0.0 20250101000000 text/plain 200 - - 0 t.arc 9
2 0 test
http://example.com/p.html 0.0.0.0 20250101000000 text/html -1 - - 0 t.arc 77
HTTP/1.1 -1 Broken
Content-Type: text/html
Content-Length: 10

11
fuzz/corpus/arc/roundtrip.arc
View File

@@ -1,11 +0,0 @@
filedesc://t.arc 0.0.0.0 20250101000000 text/plain 200 - - 0 t.arc 9
2 0 test
http://example.com/p.html 0.0.0.0 20250101000000 text/html 200 - - 0 t.arc 120
HTTP/1.1 200 OK
Content-Type: text/html
Last-Modified: Wed, 01 Jan 2025 00:00:00 GMT
Content-Length: 10
BODYMARKER

9
fuzz/corpus/arc/truncated.arc
View File

@@ -1,9 +0,0 @@
filedesc://t.arc 0.0.0.0 20250101000000 text/plain 200 - - 0 t.arc 9
2 0 test
http://example.com/q.html 0.0.0.0 20250101000000 text/html 200 - - 0 t.arc 2000000000
HTTP/1.1 200 OK
Content-Type: text/html
HI

7
fuzz/corpus/cachendx/new-format.txt
View File

@@ -1,7 +0,0 @@
8
CACHE-1.1
28
Mon, 01 Jan 2024 00:00:00 GMT
www.example.com
/index.html
123

5
fuzz/corpus/cachendx/old-format.txt
View File

@@ -1,5 +0,0 @@
3
1.0
www.example.com
/page
5

2
fuzz/corpus/cachendx/regress-overadvance.bin
View File

@@ -1,2 +0,0 @@
32768
CACHE-1.1

5
fuzz/corpus/cachendx/regress-truncated-entry.bin
View File

@@ -1,5 +0,0 @@
8
CACHE-1.1
1
x
www.example.com

1
fuzz/corpus/charset/latin1.txt
View File

@@ -1 +0,0 @@
café naďve ¤

1
fuzz/corpus/charset/sjis.txt
View File

@@ -1 +0,0 @@
コンピュータ

BIN
fuzz/corpus/charset/utf8.txt
View File

Binary file not shown.

1
fuzz/corpus/entities/entities.txt
View File

@@ -1 +0,0 @@
&amp;&lt;&gt;&#65;&#x2603;&eacute;&notarealentity;&#xFFFFFFFF;

BIN
fuzz/corpus/filters/filter-size.bin
View File

Binary file not shown.

BIN
fuzz/corpus/filters/filter.bin
View File

Binary file not shown.

BIN
fuzz/corpus/filters/redos-star-classes.bin
View File

Binary file not shown.

BIN
fuzz/corpus/filters/regress-classdepth-timeout.bin
View File

Binary file not shown.

1
fuzz/corpus/filters/regress-empty-subject-unique.bin
View File

@@ -1 +0,0 @@
**((

10
fuzz/corpus/header/full-response.txt
View File

@@ -1,10 +0,0 @@
HTTP/1.1 200 OK
Content-Type: text/html; charset=utf-8
Content-Length: 1234
Content-Encoding: gzip
Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
Etag: "abc"
Location: http://example.com/x
Set-Cookie: ID=42; path=/; domain=.example.com
Content-Range: bytes 0-99/100
Transfer-Encoding: chunked

3
fuzz/corpus/header/redirect.txt
View File

@@ -1,3 +0,0 @@
HTTP/1.0 301 Moved
Location: /elsewhere
Content-Disposition: attachment; filename="a.pdf"

15
fuzz/corpus/htsparse/basic.html
View File

@@ -1,15 +0,0 @@
<!DOCTYPE html>
<html><head>
<meta charset="utf-8">
<base href="http://example.com/dir/">
<title>Seed</title>
<link rel="stylesheet" href="style.css">
<script src="app.js"></script>
</head><body>
<h1>Hi</h1>
<a href="page2.html">next</a>
<a href="http://other.example.org/x?y=1#frag">abs</a>
<img src="pic.png" srcset="a.png 1x, b.png 2x">
<script>var u="inline.html"; document.write('<a href="gen.html">g</a>');</script>
<form action="/cgi/submit"><input name="q"></form>
</body></html>

8
fuzz/corpus/htsparse/malformed.html
View File

@@ -1,8 +0,0 @@
<a href="unclosed.html
<img src='mix"ed.png>
<a href=noquote.html >bare</a>
<!-- comment <a href="incomment.html"> -->
<a href="tab newline
.html">wsp</a>
<script>unterminated "string and <a href=
<a href=

11
fuzz/corpus/htsparse/meta-usemap.html
View File

@@ -1,11 +0,0 @@
<html><head>
<meta http-equiv="refresh" content="0; url=redir.html">
<base href="http://h/b/">
<link rel="alternate" href="feed.xml">
</head><body background="page-bg.jpg">
<img src="map.png" usemap="#m">
<map name="m"><area href="area1.html" coords="0,0,10,10"></map>
<applet code="A.class" codebase="applets/"><param name="src" value="p.dat"></applet>
<object data="o.swf"><embed src="e.svg"></object>
<a href="&#104;ttp://ent/e.html">entity</a>
</body></html>

13
fuzz/corpus/htsparse/script-inscript.html
View File

@@ -1,13 +0,0 @@
<html><body>
<script type="text/javascript">
var a = "http://x/1.html", b = 'q\'uote', c = "/*not*/comment";
// line "with" quotes and http://y/2.html
/* block 'with' <a href="notparsed"> */
var re = /a\/b\//g;
document.write('<img src="w1.png">');
document.writeln("<a href='w2.html'>k</a>");
element.onclick = "location='onh.html'";
</script>
<a href="after.html" onmouseover="go('ev.html')">x</a>
<div style="background:url(bg.png)">z</div>
</body></html>

1
fuzz/corpus/idna/idna.txt
View File

@@ -1 +0,0 @@
xn--bcher-kva.example.com

1
fuzz/corpus/idna/regress-multilabel-leak.txt
View File

@@ -1 +0,0 @@
büchev.例å­bücheple

1
fuzz/corpus/idna/unicode.txt
View File

@@ -1 +0,0 @@
bücher.例子.example

1
fuzz/corpus/meta/meta-charset.html
View File

@@ -1 +0,0 @@
<html><head><meta charset="utf-8"></head><body>x</body></html>

1
fuzz/corpus/meta/meta-content-first.html
View File

@@ -1 +0,0 @@
<meta content="text/html; charset=gb2312" http-equiv="content-type">

1
fuzz/corpus/meta/meta-http-equiv.html
View File

@@ -1 +0,0 @@
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

1
fuzz/corpus/meta/meta-truncated.html
View File

@@ -1 +0,0 @@
<meta charset='utf-8

1
fuzz/corpus/meta/meta-unquoted.html
View File

@@ -1 +0,0 @@
<meta charset = utf-8 />

3
fuzz/corpus/singlefile/img-src.html
View File

@@ -1,3 +0,0 @@
<img src="a.png">
<img src=big.png alt=over-cap>
<img src="../escape.png"><img src="/abs.png"><img src="data:,x">

4
fuzz/corpus/singlefile/link-rel.html
View File

@@ -1,4 +0,0 @@
<link rel="stylesheet" href="s.css">
<link rel=icon href=a.png>
<link rel="next" href="p2.html">
<link rel="preload" href="j.js">

5
fuzz/corpus/singlefile/malformed.html
View File

@@ -1,5 +0,0 @@
<img src="unterminated.png
<div style="background:url(a.png">
<style>@import url(
<!-- unterminated comment
<a href=

1
fuzz/corpus/singlefile/many-attrs.html
View File

@@ -1 +0,0 @@
<img src="a.png" a0="v" a1="v" a2="v" a3="v" a4="v" a5="v" a6="v" a7="v" a8="v" a9="v" a10="v" a11="v" a12="v" a13="v" a14="v" a15="v" a16="v" a17="v" a18="v" a19="v" a20="v" a21="v" a22="v" a23="v" a24="v" a25="v" a26="v" a27="v" a28="v" a29="v" a30="v" a31="v" a32="v" a33="v" a34="v" a35="v" a36="v" a37="v" a38="v" a39="v" a40="v" a41="v" a42="v" a43="v" a44="v" a45="v" a46="v" a47="v" a48="v" a49="v" a50="v" a51="v" a52="v" a53="v" a54="v" a55="v" a56="v" a57="v" a58="v" a59="v" a60="v" a61="v" a62="v" a63="v" a64="v" a65="v" a66="v" a67="v" a68="v" a69="v">

4
fuzz/corpus/singlefile/rawtext.html
View File

@@ -1,4 +0,0 @@
<script>var s="</scripting>"; if(a</b) x=1;</script>
<script src="j.js"></script>
<textarea></textareas></textarea>
<title></titles></title>

2
fuzz/corpus/singlefile/srcset.html
View File

@@ -1,2 +0,0 @@
<img srcset="a.png 1x, big.png 2x, a.png 100w">
<source srcset="a.png,, a.png 2x," src="a.png">

2
fuzz/corpus/singlefile/style-attr.html
View File

@@ -1,2 +0,0 @@
<div style="background:url(a.png);list-style:url('a.png')"></div>
<p style='background:url("a.png")'>x</p>

5
fuzz/corpus/singlefile/style-block.html
View File

@@ -1,5 +0,0 @@
<style>@import "s.css";
@import url(sub/b.css);
div{background:url(a.png)}
/* url(a.png) */ p:after{content:"url(a.png)"}
</style>

1
fuzz/corpus/sitemap/sitemapindex.xml
View File

@@ -1 +0,0 @@
<sitemapindex><sitemap><loc>http://h.test/s2.xml.gz</loc></sitemap></sitemapindex>

1
fuzz/corpus/sitemap/truncated.xml
View File

@@ -1 +0,0 @@
<urlset><loc>http://h.test/x

1
fuzz/corpus/sitemap/urlset.xml
View File

@@ -1 +0,0 @@
<?xml version="1.0"?><urlset><url><loc>http://h.test/a.html</loc></url><url><loc>https://h.test/b?x=1&amp;y=2</loc></url></urlset>

BIN
fuzz/corpus/sitemap/urlset.xml.gz
View File

Binary file not shown.

1
fuzz/corpus/unescape/percent.txt
View File

@@ -1 +0,0 @@
%41%zz%%20%c3%a9+%2e%2e%2f

1
fuzz/corpus/url/http-url.txt
View File

@@ -1 +0,0 @@
http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

1
fuzz/corpus/url/regress-file-empty-path.txt
View File

@@ -1 +0,0 @@
file://

1
fuzz/corpus/url/regress-long-path-abort.txt
View File

@@ -1 +0,0 @@
ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

1
fuzz/corpus/url/relative-path.txt
View File

@@ -1 +0,0 @@
ftp://ftp.example.com/pub/../file.txt

106
fuzz/fuzz-arc.c
View File

@@ -1,106 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz proxytrack's .arc reader the way `--convert` drives it: the record loop
seeks on lengths read from the file, and every entry reaches a writer. */
#include "fuzz.h"
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "coucal.h"
#include "proxy/store.h"
/* An .arc past this size says nothing a smaller one cannot. */
#define FUZZ_ARC_MAXSIZE (1024 * 1024)
static char arc_dir[256];
static char arc_in[sizeof(arc_dir) + sizeof("/in.arc")];
static char arc_out[sizeof(arc_dir) + sizeof("/out.arc")];
static void fuzz_arc_cleanup(void) {
(void) unlink(arc_in);
(void) unlink(arc_out);
(void) rmdir(arc_dir);
}
/* proxytrack's main() installs one; without it coucal logs stats per free */
static void fuzz_arc_coucal_log(coucal_opaque arg, coucal_loglevel level,
const char *format, va_list args) {
(void) arg;
(void) level;
(void) format;
(void) args;
}
/* PT_GetType() picks the format from the extension, so the names end in .arc */
static int fuzz_arc_setup(void) {
if (arc_in[0] == '\0') {
const char *const tmp = getenv("TMPDIR");
coucal_set_global_assert_handler(fuzz_arc_coucal_log, NULL);
snprintf(arc_dir, sizeof(arc_dir), "%s/fuzz-arc-XXXXXX",
tmp != NULL && *tmp != '\0' ? tmp : "/tmp");
if (mkdtemp(arc_dir) == NULL) {
return -1;
}
snprintf(arc_out, sizeof(arc_out), "%s/out.arc", arc_dir);
snprintf(arc_in, sizeof(arc_in), "%s/in.arc", arc_dir);
atexit(fuzz_arc_cleanup);
}
return 0;
}
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
PT_Indexes indexes;
FILE *fp;
hts_boolean written;
if (size > FUZZ_ARC_MAXSIZE || fuzz_arc_setup() != 0) {
return 0;
}
if ((fp = fopen(arc_in, "wb")) == NULL) {
return 0;
}
written = fwrite(data, 1, size, fp) == size ? HTS_TRUE : HTS_FALSE;
if (fclose(fp) != 0 || !written) {
return 0;
}
indexes = PT_New();
if (indexes != NULL) {
if (PT_AddIndex(indexes, arc_in) > 0) {
/* the writer reads back every entry the loader indexed */
(void) PT_SaveCache(indexes, arc_out);
}
PT_Delete(indexes);
}
return 0;
}

65
fuzz/fuzz-cachendx.c
View File

@@ -1,65 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
Mirrors the -#C cache-listing scan (htscoremain.c). */
#include "fuzz.h"
#include "htscache.h"
#include "htslib.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
char *buf = fuzz_strdup(data, size);
const char *const end = buf + size;
char firstline[256];
char *a = buf;
/* header: two length-prefixed fields (version, last-modified) */
a += cache_brstr(a, firstline, sizeof(firstline));
a += cache_brstr(a, firstline, sizeof(firstline));
/* body: newline-delimited host/file/position triples; the length-prefixed
scan must stay inside the buffer */
while (a != NULL && a < end) {
char BIGSTK line[HTS_URLMAXSIZE * 2];
char linepos[256];
int pos;
a = strchr(a + 1, '\n');
if (a == NULL)
break;
a++;
a += cache_binput(a, end, line, HTS_URLMAXSIZE);
a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
a += cache_binput(a, end, linepos, 200);
sscanf(linepos, "%d", &pos);
(void) pos;
}
freet(buf);
return 0;
}

84
fuzz/fuzz-charset.c
View File

@@ -1,84 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
#include "fuzz.h"
#include "htscharset.h"
static const char *const charsets[] = {
"utf-8", "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
"us-ascii", "shift_jis", "euc-jp", "iso-2022-jp", "gb2312",
"big5", "euc-kr", "koi8-r", "utf-16", "unknown-charset",
};
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
const char *charset;
char *s;
if (size == 0)
return 0;
charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
data++, size--;
s = fuzz_strdup(data, size);
{
char *utf8 = hts_convertStringToUTF8(s, size, charset);
freet(utf8);
}
{
char *enc = hts_convertStringFromUTF8(s, size, charset);
freet(enc);
}
{
size_t nChars = 0;
hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
if (ucs != NULL) {
char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
freet(back);
freet(ucs);
}
}
{
size_t i = 0;
while (i < size) {
hts_UCS4 uc = 0;
const size_t nr = hts_readUTF8(s + i, size - i, &uc);
char out[8];
if (nr == 0)
break;
hts_writeUTF8(uc, out, sizeof(out));
i += nr;
}
}
freet(s);
return 0;
}

51
fuzz/fuzz-entities.c
View File

@@ -1,51 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
destination size, so truncation bounds get exercised too. */
#include "fuzz.h"
#include "htsencoding.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
static const size_t dsizes[] = {1, 2, 8, 64, 4096};
size_t dsize;
char *src, *dest;
if (size == 0)
return 0;
dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
data++, size--;
src = fuzz_strdup(data, size);
dest = malloct(dsize);
(void) hts_unescapeEntities(src, dest, dsize);
(void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
freet(dest);
freet(src);
return 0;
}

65
fuzz/fuzz-filters.c
View File

@@ -1,65 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
here). Input splits on the first NUL: pattern, then subject string. */
#include "fuzz.h"
#include "htsfilters.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
char *buf = fuzz_strdup(data, size);
const char *joker = buf;
const uint8_t *sep = memchr(data, '\0', size);
/* subject in its own allocation so ASan bounds it apart from the pattern */
char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
: fuzz_strdup(data + size, 0);
(void) strjoker(nom, joker, NULL, NULL);
{
LLint sz = (LLint) size;
int size_flag = 0;
(void) strjoker(nom, joker, &sz, &size_flag);
}
(void) strjokerfind(nom, joker);
{
char *filter = malloct(strlen(joker) + 2);
char *filters[1];
LLint sz = (LLint) size;
int size_flag = 0, depth = 0;
filter[0] = '-';
memcpy(filter + 1, joker, strlen(joker) + 1);
filters[0] = filter;
(void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
freet(filter);
}
freet(nom);
freet(buf);
return 0;
}

74
fuzz/fuzz-header.c
View File

@@ -1,74 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
status line, treathead on each following header. Both consume raw bytes
off the wire and copy fields into fixed htsblk buffers; treathead also
mutates its line in place and drives cookie parsing. */
#include "fuzz.h"
#include "htslib.h"
#include "htsbauth.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
char *buf = fuzz_strdup(data, size);
htsblk r;
t_cookie *cookie = calloct(1, sizeof(*cookie));
char *line = malloct(size + 1);
char *p = buf;
int first = 1;
memset(&r, 0, sizeof(r));
cookie->max_len = (int) sizeof(cookie->data);
r.location = malloct(HTS_URLMAXSIZE * 2);
r.location[0] = '\0';
/* feed one header line at a time, as the receive loop does */
while (p != NULL && *p != '\0') {
char *nl = strchr(p, '\n');
size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
size_t i, len = 0;
/* binput drops every '\r' on the wire; mirror it */
for (i = 0; i < n; i++)
if (p[i] != '\r')
line[len++] = p[i];
line[len] = '\0';
if (first) {
treatfirstline(&r, line);
first = 0;
} else {
treathead(cookie, "www.example.com", "/", &r, line);
}
p = (nl != NULL) ? nl + 1 : NULL;
}
freet(r.location);
freet(line);
freet(cookie);
freet(buf);
return 0;
}

175
fuzz/fuzz-htsparse.c
View File

@@ -1,175 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the real htsparse() over a mocked engine: the minimal crawl state
httpmirror() builds, then a page walked through the parser and discarded. The
str/stre wiring below mirrors htsparse()'s call site in htscore.c; update it
in lockstep if those structs gain a field the parser reads. */
#include "fuzz.h"
#include "httrack-library.h"
#include "htscore.h"
#include "htsback.h"
#include "htshash.h"
#include "htsrobots.h"
#include "htsparse.h"
#include "htsmodules.h"
#include "coucal.h"
/* htsparse ignores str.addLink on the internal parse; stub it. */
static int fuzz_addlink(htsmoduleStruct *str, char *link) {
(void) str;
(void) link;
return 0;
}
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
static int inited = 0;
httrackp *opt;
cache_back cache;
hash_struct hash;
robots_wizard robots;
struct_back *sback;
char **filters = NULL;
int filptr = 0;
htsblk r;
htsmoduleStruct str;
htsmoduleStructExtended stre;
int ptr, error = 0, store_errpage = 0;
int makeindex_done = 0, makeindex_links = 0;
FILE *makeindex_fp = NULL;
char makeindex_firstlink[HTS_URLMAXSIZE * 2] = "";
LLint stat_fragment = 0, makestat_total = 0;
int makestat_lnk = 0;
char base[HTS_URLMAXSIZE * 2] = "";
char codebase[HTS_URLMAXSIZE * 2] = "";
char err_msg[1024] = "";
if (!inited) {
hts_init();
inited = 1;
}
opt = hts_create_opt();
opt->log = opt->errlog = NULL;
opt->robots = 0;
memset(&cache, 0, sizeof(cache));
cache.type = 0; /* no on-disk cache */
cache.hashtable = coucal_new(0);
cache.cached_tests = coucal_new(0);
coucal_value_is_malloc(cache.cached_tests, 1);
memset(&robots, 0, sizeof(robots));
strcpybuff(robots.adr, "!");
opt->robotsptr = &robots;
opt->maxfilter = maximum(opt->maxfilter, 128);
filters_init(&filters, opt->maxfilter, 0);
opt->filters.filters = &filters;
opt->filters.filptr = &filptr;
opt->hash = &hash;
hts_record_init(opt);
hash_init(opt, &hash, opt->urlhack);
hash.liens = (const lien_url *const *const *) &opt->liens;
sback = back_new(opt, opt->maxsoc * 32 + 1024);
/* ptr=1 (index 1 is the parsed page) selects full HTML parsing + the
rewriter; urladr()/urlfil()/savename() alias heap(ptr), save=/dev/null. */
hts_record_link(opt, "example.com", "/", "/dev/null", "", "", NULL);
hts_record_link(opt, "example.com", "/index.html", "/dev/null", "", "", NULL);
ptr = 1;
/* NUL-terminated in a size+1 alloc: htsparse one-past-reads onto the NUL. */
hts_init_htsblk(&r);
r.statuscode = 200;
r.size = (LLint) size;
r.adr = malloct(size + 1);
if (size)
memcpy(r.adr, data, size);
r.adr[size] = '\0';
strcpybuff(r.contenttype, "text/html");
memset(&str, 0, sizeof(str));
memset(&stre, 0, sizeof(stre));
str.err_msg = err_msg;
str.filename = heap(ptr)->sav;
str.mime = r.contenttype;
str.url_host = heap(ptr)->adr;
str.url_file = heap(ptr)->fil;
str.size = (int) r.size;
str.addLink = fuzz_addlink;
str.opt = opt;
str.sback = sback;
str.cache = &cache;
str.hashptr = &hash;
str.numero_passe = 0;
str.ptr_ = &ptr;
str.page_charset_ = NULL;
stre.r_ = &r;
stre.error_ = &error;
stre.exit_xh_ = &opt->state.exit_xh;
stre.store_errpage_ = &store_errpage;
stre.base = base;
stre.codebase = codebase;
stre.filters_ = &filters;
stre.filptr_ = &filptr;
stre.robots_ = &robots;
stre.hash_ = &hash;
stre.makeindex_done_ = &makeindex_done;
stre.makeindex_fp_ = &makeindex_fp;
stre.makeindex_links_ = &makeindex_links;
stre.makeindex_firstlink_ = makeindex_firstlink;
stre.template_header_ = "";
stre.template_body_ = "";
stre.template_footer_ = "";
stre.stat_fragment_ = &stat_fragment;
stre.makestat_time = 0;
stre.makestat_fp = NULL;
stre.makestat_total_ = &makestat_total;
stre.makestat_lnk_ = &makestat_lnk;
stre.maketrack_fp = NULL;
(void) htsparse(&str, &stre);
freet(r.adr);
back_delete_all(opt, &cache, sback);
back_free(&sback);
hash_free(&hash);
coucal_delete(&cache.hashtable);
coucal_delete(&cache.cached_tests);
checkrobots_free(&robots);
if (filters != NULL) {
if (filters[0] != NULL)
freet(filters[0]);
freet(filters);
}
hts_record_free(opt);
hts_free_opt(opt);
return 0;
}

47
fuzz/fuzz-idna.c
View File

@@ -1,47 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
#include "fuzz.h"
#include "htscharset.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
char *s = fuzz_strdup(data, size);
{
char *idna = hts_convertStringUTF8ToIDNA(s, size);
freet(idna);
}
{
char *utf8 = hts_convertStringIDNAToUTF8(s, size);
freet(utf8);
}
(void) hts_isStringIDNA(s, size);
freet(s);
return 0;
}

40
fuzz/fuzz-meta.c
View File

@@ -1,40 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
<meta> charset declaration. */
#include "fuzz.h"
#include "htscharset.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
char *html = fuzz_strdup(data, size);
char *charset = hts_getCharsetFromMeta(html, size);
freet(charset);
freet(html);
return 0;
}

131
fuzz/fuzz-singlefile.c
View File

@@ -1,131 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the --single-file rewriter (htssinglefile.c): hostile HTML walked
through the tag, CSS url()/@import and srcset parsers, then re-serialized.
The resolver is aimed at a private temp tree, so the inlining half (MIME
guess, base64, nested stylesheet) is reached and nothing else on disk is. */
#include "fuzz.h"
#include "httrack-library.h"
#include "htssinglefile.h"
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
/* Between a.png and big.png, so one input reaches both the inline path and the
over-cap fallback. */
#define FUZZ_SF_CAP 64
static char sf_root[512];
static char sf_page[600];
/* The asset tree, in removal order: the subdirectory comes after its file. */
static const char *const sf_files[] = {"a.png", "big.png", "j.js", "s.css",
"sub/b.css", "sub", NULL};
static void sf_cleanup(void) {
char path[700];
int i;
for (i = 0; sf_files[i] != NULL; i++) {
snprintf(path, sizeof(path), "%s/%s", sf_root, sf_files[i]);
(void) remove(path);
}
(void) remove(sf_root);
}
/* A missing asset would silently reduce the target to its parser half. */
static void sf_write(const char *name, const char *data, size_t len) {
char path[700];
FILE *fp;
snprintf(path, sizeof(path), "%s/%s", sf_root, name);
fp = fopen(path, "wb");
if (fp == NULL || fwrite(data, 1, len, fp) != len)
abort();
fclose(fp);
}
static void sf_text(const char *name, const char *data) {
sf_write(name, data, strlen(data));
}
static void sf_init(void) {
static const char png[] = "\x89PNG\r\n\x1a\n";
static const char big[4096] = "\x89PNG";
const char *tmp = getenv("TMPDIR");
char path[700];
hts_init();
snprintf(sf_root, sizeof(sf_root), "%s/httrack-fuzz-sf-XXXXXX",
tmp != NULL && tmp[0] != '\0' ? tmp : "/tmp");
if (mkdtemp(sf_root) == NULL)
abort();
atexit(sf_cleanup);
snprintf(sf_page, sizeof(sf_page), "%s/page.html", sf_root);
snprintf(path, sizeof(path), "%s/sub", sf_root);
if (mkdir(path, 0700) != 0)
abort();
sf_write("a.png", png, sizeof(png) - 1);
sf_write("big.png", big, sizeof(big));
sf_text("j.js", "var x=1;\n");
/* @import plus a url(), so an inlined stylesheet recurses and its own
relative reference is rebased. */
sf_text("s.css", "@import url(sub/b.css);\ndiv{background:url(a.png)}\n");
sf_text("sub/b.css", "p{background:url(../a.png)}\n");
}
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
static int inited = 0;
String out = STRING_EMPTY;
httrackp *opt;
/* Exact-length, unterminated: the rewriter is span-based, so ASan bounds a
read past html_len instead of it landing on a terminator. */
char *html = malloct(size != 0 ? size : 1);
if (!inited) {
sf_init();
inited = 1;
}
memcpy(html, data, size);
opt = hts_create_opt();
opt->log = opt->errlog = NULL;
opt->single_file_max_size = FUZZ_SF_CAP;
StringClear(out);
(void) singlefile_rewrite_html(opt, sf_root, sf_page, html, size,
SINGLEFILE_MAX_PAGE_SIZE, &out);
StringFree(out);
freet(html);
hts_free_opt(opt);
return 0;
}

60
fuzz/fuzz-sitemap.c
View File

@@ -1,60 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the sitemap <loc> scanner (htssitemap.c): raw XML, gzip-framed bodies
and truncated streams all arrive here straight off the network. */
#include "fuzz.h"
#include "htssitemap.h"
static hts_boolean sm_count(void *arg, const char *url) {
int *const n = (int *) arg;
(void) url;
(*n)++;
return HTS_TRUE;
}
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
static const int caps[] = {0, 1, 16, HTS_SITEMAP_MAX_URLS_DOC};
hts_boolean is_index;
char *body;
int n = 0, cap;
if (size == 0)
return 0;
cap = caps[data[0] % (sizeof(caps) / sizeof(caps[0]))];
data++, size--;
/* A heap copy of exactly `size` bytes: the scanner must never rely on a
terminator, and ASan turns any overread into a report. */
body = malloct(size != 0 ? size : 1);
memcpy(body, data, size);
(void) hts_sitemap_scan(body, size, cap, &is_index, sm_count, &n);
freet(body);
return 0;
}

53
fuzz/fuzz-unescape.c
View File

@@ -1,53 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
output buffer size, so the bounded-copy contract is exercised. */
#include "fuzz.h"
#include "httrack-library.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
static const size_t bsizes[] = {1, 2, 16, 256, 8192};
size_t bsize;
char *s, *catbuff;
if (size == 0)
return 0;
bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
data++, size--;
s = fuzz_strdup(data, size);
catbuff = malloct(bsize);
(void) unescape_http(catbuff, bsize, s);
(void) unescape_http_unharm(catbuff, bsize, s, 0);
(void) unescape_http_unharm(catbuff, bsize, s, 1);
unescape_amp(s);
freet(catbuff);
freet(s);
return 0;
}

49
fuzz/fuzz-url.c
View File

@@ -1,49 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
in place. */
#include "fuzz.h"
#include "htscore.h"
#include "htslib.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
char *s = fuzz_strdup(data, size);
lien_adrfil *af = calloct(1, sizeof(*af));
/* fil_simplifie rewrites in place and may grow an empty path to "./" */
char *path = malloct(size + 3);
(void) ident_url_absolute(s, af);
memcpy(path, s, size + 1);
fil_simplifie(path);
freet(path);
freet(af);
freet(s);
return 0;
}

51
fuzz/fuzz.h
View File

@@ -1,51 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* Shared helpers for the libFuzzer harnesses. */
#ifndef FUZZ_H
#define FUZZ_H
#define HTS_INTERNAL_BYTECODE
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include "htsbase.h"
/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
HTS_UNUSED static char *fuzz_strdup(const uint8_t *data, size_t size) {
char *s = malloct(size + 1);
memcpy(s, data, size);
s[size] = '\0';
return s;
}
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
#endif

41
fuzz/run-fuzzers.sh
View File

@@ -1,41 +0,0 @@
#!/bin/bash
# Drive every built harness against its seed corpus.
# run-fuzzers.sh <build-fuzz-dir> check deterministic replay (CI smoke)
# run-fuzzers.sh <build-fuzz-dir> [seconds] timed mutation run (discovery)
# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
set -euo pipefail
srcdir=$(cd "$(dirname "$0")" && pwd)
bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
mode=${2:-20}
status=0
for f in "$bld"/fuzz-*; do
if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
case "$f" in *.o | *.c | *.dSYM) continue ;; esac
name=$(basename "$f")
corpus="$srcdir/corpus/${name#fuzz-}"
if [ "$mode" = "check" ]; then
echo "=== $name (replay) ==="
[ -d "$corpus" ] || continue
if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
echo "*** $name FAILED on its corpus" >&2
status=1
fi
continue
fi
work=$(mktemp -d)
args=("$work")
[ -d "$corpus" ] && args+=("$corpus")
echo "=== $name (${mode}s) ==="
if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
-artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
echo "*** $name FAILED; artifacts:" >&2
ls -l "$work" >&2
status=1
else
rm -rf "$work"
fi
done
exit $status

2
greetings.txt
View File

@@ -5,7 +5,7 @@ Xavier Roche (xroche at httrack.com)
project leader
core engine, Windows/Linux GUI
Yann Philippot (yphilippot at lemel.fr)
past contributor (java binary .class parser)
for the java binary .class parser
With the help of:
Leto Kauler (molotov at tasmail.com)

108
history.txt
View File

@@ -4,112 +4,6 @@ HTTrack Website Copier release history:
This file lists all changes and fixes that have been made for HTTrack
3.49-15
+ New: --single-file rewrites each saved page with its assets inlined as data: URIs (#713)
+ New: --changes reports what a crawl added, updated or removed against the previous mirror (#714)
+ New: --sitemap and --sitemap-url ingest sitemaps, so pages nothing links to are still found (#712)
+ New: webhttrack exposes --warc-cdx, --wacz and --warc-max-size (#862)
+ Fixed: --update and --purge-old destroyed a good local copy when the re-fetch got no response, was aborted mid-read, or when the backup meant to protect it failed (#746, #748, #758, #775)
+ Fixed: an FTP re-fetch truncated the mirrored file, resumed a complete mirror with REST and spliced the old body into the new one, and a successful transfer was blanked when its backlog slot was swapped out (#771, #797, #798, #823)
+ Fixed: a chunked response cut at a chunk boundary was stored and cached as complete (#840)
+ Fixed: cache repair deleted the old cache before a rename it never checked, and the unlink-then-rename fallback could lose the destination (#779, #786, #790, #824)
+ Fixed: several backward scans from strlen(s) - 1 read before their buffer on an empty string (#730, #768, #770, #814, #821)
+ Fixed: a URL could be saved onto the engine's own temporary files, and the final path segment was never clamped (#774, #842, #852)
+ Fixed: a query-string character reference the page charset cannot represent was left unescaped, changing how the query parses (#854)
+ Fixed: a second --update pass overwrote the previous WARC and regenerated a page-less WACZ (#759)
+ Fixed: WARC output dropped URLs of 1005 bytes or more, archived nothing for a 304 revisit, and marked an engine-forced not-modified as one the server sent (#778, #785, #826, #838, #839)
+ Fixed: ProxyTrack overflowed its .arc header block, walked past the end of an .ndx buffer, trusted an unparsed offset, and crashed on a PROPFIND or on an entry with no usable Last-Modified (#793, #820, #825, #828)
+ Fixed: webhttrack leaked the session id into crawled pages, overflowed its command line so a quoted value could inject flags, let a posted project path repoint the served root, and built its redirect Location in a 256-byte stack buffer (#700, #706, #707, #710)
+ Fixed: in the web GUI, options ticked on by default could not be un-ticked, "max site size" set a per-file cap instead of the HTML one, and the mirror link on the finished page could not be followed (#708, #709, #725)
+ Fixed: htsserver labelled every PNG as image/gif and offered JPEG as a download, and an unauthenticated GET of a directory spun the server forever (#724, #875)
+ Fixed: webhttrack hung at exit when no mirror had been launched (#753)
+ Fixed: oversized cache and header fields aborted the engine or overflowed a neighbouring field instead of being clipped (#701, #715, #717, #722, #732)
+ Fixed: a -%S rules file of 4 GB or more overran the heap (#702)
+ Fixed: the CLI display did not repaint when the terminal was resized (#97)
+ Fixed: a document with no declared charset double-encoded the title lifted for the local index (#848)
+ Fixed: a fragment on an inlined reference was dropped, losing an SVG sprite selector (#766)
+ Fixed: several time helpers handed out libc's shared gmtime/localtime static rather than a reentrant breakdown (#794, #806)
+ Changed: fatal-signal backtraces name engine frames instead of a bare module and offset (#705)
+ Changed: --without-zlib is rejected at configure time rather than failing at link (#735)
+ Changed: the offline documentation gains one GUI guide with screenshots, an Android option reference, and a restructured index
+ Changed: multiple internal hardening, test and CI improvements
3.49-14
+ New: WARC/1.1 archive output (--warc), with a sorted CDXJ index (--warc-cdx) and WACZ packaging (--wacz), also available from webhttrack (#668)
+ New: -%F takes named footer fields such as {url}, {lastmodified}, {mime}, {charset} and {status} instead of a fixed layout (#667)
+ New: a command-line guide organized by task ships with the offline documentation (#649)
+ Fixed: on Windows, paths beyond MAX_PATH truncated files, and mirroring into a long or non-ASCII directory silently failed (#133)
+ Fixed: the top index showed mojibake for non-ASCII project names and categories on Windows (#216)
+ Fixed: webhttrack handed the engine the web form's charset rather than UTF-8, so a non-ASCII path mirrored into a mojibake directory (#629)
+ Fixed: a non-ASCII single -O left the logs and the cache in a mangled twin directory on Windows (#630)
+ Fixed: a path-ceiling truncation dropped the .delayed marker, losing the file (#623)
+ Fixed: an oversized -%F footer aborted the crawl instead of being skipped (#669)
+ Fixed: a rejected 206 resume could loop and lose the file rather than refetch it whole (#581)
+ Fixed: default-port stripping was scheme-blind and dropped explicit ports from https and ftp URLs, and a :80 written with leading zeros mangled the host (#627, #638)
+ Fixed: -K silently reset the -c socket count (#650)
+ Fixed: signed-shift undefined behaviour in the zip-repair local-header read (#639)
+ Changed: the offline documentation drops stale facts, gains an Android help page, and documents the filter wildcards and the real long option forms
+ Changed: multiple internal hardening, test and CI improvements
3.49-13
+ New: SOCKS5 proxy support, with scheme-aware -P URLs (socks5://, socks5h://, connect://) and plain HTTP tunneled through a CONNECT-only proxy (#563, #564)
+ New: decode brotli and zstd content codings, advertised over TLS only as browsers do (#556)
+ New: webhttrack exposes the engine options added since 3.49-2, among them --cookies-file, --pause and --strip-query (#587)
+ Fixed: files of 2 GB or more were mishandled on Windows and on every 32-bit build (#569)
+ Fixed: --update destroyed a good local copy when the re-fetch returned an HTTP error, was aborted by -M/-E, failed to decode, or came in short (#176, #521, #557, #562)
+ Fixed: a self-redirect cookie wall was dropped instead of being re-fetched with the cookie (#15)
+ Fixed: a stalled TLS handshake ignored --timeout, and synchronous DNS resolution could wedge a crawl past --max-time (#607, #613)
+ Fixed: -M metered saved bytes rather than received volume, and overshot its cap under a slow server (#77, #520)
+ Fixed: several network-facing overflows and denial-of-service paths in the Content-Range, chunked-transfer, cookie, filter and ProxyTrack cache parsers
+ Fixed: a failed connect did not fall back to the next address on Windows (#579)
+ Fixed: -P took an out-of-range port as a garbage port, and scanned past an IPv6 literal's closing bracket (#598, #602)
+ Fixed: reject a port outside 1..65535 wherever one is parsed (a crawled URL, the htsserver and proxytrack listen arguments, an ftp:// URL), instead of letting a bare sscanf wrap a huge value into a plausible port and silently use it (#614, #626)
+ Fixed: a configured proxy still resolved and dialed the origin itself (#592)
+ Fixed: ~/ in the -O base path was never expanded (#270)
+ Fixed: a non-ASCII -O output path was double-encoded on Windows once argv became UTF-8 (#621)
+ Fixed: files under a non-ASCII project path were saved to a mangled directory on Windows (#217)
+ Fixed: --build-top-index (-%i) and --protocol (-@i) were taken for the -i continue flag, wiping the URL list and exiting on the usage screen (#615)
+ Fixed: webhttrack ignored LC_ALL/LC_MESSAGES and picked the wrong Chinese and Portuguese (#95)
+ Fixed: webhttrack wrote its base path and httrack.ini to the filesystem root when $HOME was empty (#625)
+ Fixed: crawls on a non-default port were slowed by a per-request pre-resolve (#181)
+ Changed: Windows builds moved to Visual Studio 2022 and OpenSSL 3.x, the VS2008 project files are retired, and the binaries carry a version resource
+ Changed: removed the obsolete Java-applet (.class) parser and the dead SWF module remnants
+ Changed: multiple internal hardening, test and CI improvements (Windows and macOS crawl suites, HTML-parser fuzzing, parallel make check)
3.49-12
+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
+ Fixed: cookies.txt was created world-readable (#511)
+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
3.49-11
+ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
+ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
+ New: follow <source> and <track> media elements as embedded links (#451)
+ New: added modern web MIME types to the type/extension table (#448)
+ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
+ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
+ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
+ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
+ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
+ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
+ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
+ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
+ Changed: decode the full WHATWG set of HTML named character references (#443)
+ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
+ Changed: multiple internal hardening, build, test and CI improvements
3.49-10
+ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
+ New: --pause to space out file downloads by a random delay (#185)
@@ -128,7 +22,7 @@ This file lists all changes and fixes that have been made for HTTrack
+ Fixed: report why a -%L URL list could not be loaded (#49)
+ Changed: multiple internal hardening, build and CI improvements
3.49-9
.49-9
+ Fixed: file-type detection from the Content-Type header: trust a declared type over a binary URL extension, honor --assume under the delayed type check, and keep a known extension against a bogus or empty Content-Type (#267, #29, #56)
+ Fixed: an uninitialized-buffer read when the Content-Type is empty (#411)
+ Fixed: restored C++ source-compatibility of the installed headers so reverse dependencies (httraqt) build again (#413)

27
html/Makefile.am
View File

@@ -11,10 +11,6 @@ WebPixmapdir = $(datadir)/pixmaps
WebIcon16x16dir = $(datadir)/icons/hicolor/16x16/apps
WebIcon32x32dir = $(datadir)/icons/hicolor/32x32/apps
WebIcon48x48dir = $(datadir)/icons/hicolor/48x48/apps
WebIcon64x64dir = $(datadir)/icons/hicolor/64x64/apps
WebIcon128x128dir = $(datadir)/icons/hicolor/128x128/apps
WebIcon256x256dir = $(datadir)/icons/hicolor/256x256/apps
WebIconScalabledir = $(datadir)/icons/hicolor/scalable/apps
VFolderEntrydir = $(prefix)/share/applications
MetaInfodir = $(datadir)/metainfo
@@ -23,29 +19,32 @@ MetaInfodir = $(datadir)/metainfo
# Explicit filenames (e.g. ../history.txt, div/search.sh) resolve via VPATH and
# need no prefix.
HelpHtmlroot_DATA = ../httrack-doc.html ../history.txt
HelpHtml_DATA = $(srcdir)/*.html $(srcdir)/*.js $(srcdir)/*.css
HelpHtml_DATA = $(srcdir)/*.html
HelpHtmldiv_DATA = div/search.sh
HelpHtmlimg_DATA = $(srcdir)/img/*
HelpHtmlimages_DATA = $(srcdir)/images/*
HelpHtmlTxt_DATA = ../greetings.txt ../history.txt ../license.txt
WebHtml_DATA = $(srcdir)/server/*.html $(srcdir)/server/*.js $(srcdir)/server/*.css
WebHtmlimages_DATA = $(srcdir)/server/images/*
# Generated from the brand master by gen/export.py. Asserted by tests/160.
# httrack.xpm duplicates the 32x32 one under the bare basename the .desktop
# Icon= key falls back to here when no icon theme is installed (#932).
# note: converted & normalized by
# ico2xpm favicon.ico -o httrack.xpm
# mogrify -format xpm -map /usr/share/doc/menu/examples/cmap.xpm httrack.xpm
WebPixmap_DATA = $(srcdir)/server/div/*.xpm
WebIcon16x16_DATA = $(srcdir)/server/div/16x16/*.png
WebIcon32x32_DATA = $(srcdir)/server/div/32x32/*.png
WebIcon48x48_DATA = $(srcdir)/server/div/48x48/*.png
WebIcon64x64_DATA = $(srcdir)/server/div/64x64/*.png
WebIcon128x128_DATA = $(srcdir)/server/div/128x128/*.png
WebIcon256x256_DATA = $(srcdir)/server/div/256x256/*.png
WebIconScalable_DATA = $(srcdir)/server/div/scalable/*.svg
VFolderEntry_DATA = $(srcdir)/server/div/*.desktop
MetaInfo_DATA = $(srcdir)/server/div/*.metainfo.xml
EXTRA_DIST = $(HelpHtml_DATA) $(HelpHtmlimg_DATA) $(HelpHtmlimages_DATA) \
$(HelpHtmldiv_DATA) $(WebHtml_DATA) $(WebHtmlimages_DATA) \
$(WebPixmap_DATA) $(WebIcon16x16_DATA) $(WebIcon32x32_DATA) $(WebIcon48x48_DATA) \
$(WebIcon64x64_DATA) $(WebIcon128x128_DATA) $(WebIcon256x256_DATA) \
$(WebIconScalable_DATA) $(VFolderEntry_DATA) $(MetaInfo_DATA)
$(VFolderEntry_DATA) $(MetaInfo_DATA) \
httrack.css
install-data-hook:
if test ! -L $(DESTDIR)$(prefix)/share/httrack/html ; then \
( cd $(DESTDIR)$(prefix)/share/httrack \
&& $(LN_S) $(htmldir) html \
) \
fi

241
html/abuse.html
View File

@@ -1,74 +1,111 @@
<!DOCTYPE html>
<html lang="en">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<!-- doc-chrome:head -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="How to mirror a site without hammering the server, and what a webmaster can do about crawlers that do.">
<link rel="stylesheet" href="doc.css">
<script src="doc.js" defer></script>
<!-- /doc-chrome:head -->
<title>Using HTTrack responsibly</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
<title>HTTrack Website Copier - Offline Browser</title>
<style type="text/css">
<!--
body {
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
background: #77b;
}
body, td {
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
}
#subTitle {
background: #000; color: #fff; padding: 4px; font-weight: bold;
}
#siteNavigation a, #siteNavigation .current {
font-weight: bold; color: #448;
}
#siteNavigation a:link { text-decoration: none; }
#siteNavigation a:visited { text-decoration: none; }
#siteNavigation .current { background-color: #ccd; }
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
a:link { text-decoration: underline; color: #00f; }
a:visited { text-decoration: underline; color: #000; }
a:hover { text-decoration: underline; color: #c00; }
a:active { text-decoration: underline; }
#pageContent {
clear: both;
border-bottom: 6px solid #000;
padding: 10px; padding-top: 20px;
line-height: 1.65em;
background-image: url(images/bg_rings.gif);
background-repeat: no-repeat;
background-position: top right;
}
#pageContent, #siteNavigation {
background-color: #ccd;
}
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
h1 { margin: 0; font-weight: bold; font-size: 2em; }
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
.blak { background-color: #000; }
.hide { display: none; }
.tableWidth { min-width: 400px; }
.tblRegular { border-collapse: collapse; }
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
.tblNoBorder td { border: 0; }
// -->
</style>
</head>
<body>
<!-- doc-chrome:top -->
<a class="skip" href="#main">Skip to content</a>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr>
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
<tr>
<td id="subTitle">Open Source offline browser</td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr class="blak">
<td>
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
<tr>
<td colspan="6">
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
<tr>
<td id="pageContent">
<!-- ==================== End prologue ==================== -->
<header class="masthead">
<img src="images/wordmark.svg" width="400" height="36" alt="HTTrack Website Copier">
<div class="tagline">Open Source offline browser</div>
</header>
<div class="wrap">
<nav class="toc" aria-label="Documentation">
<h2>On this page</h2>
<ul>
<li><a href="#for-httrack-users">For HTTrack users</a></li>
<li><a href="#for-webmasters-having-problems-with-band">For webmasters having problems with bandwidth abuse / other abuses related to HTTrack</a></li>
<li><a href="#please-follow-these-common-sense-rules-t">Please follow these common sense rules to avoid any network abuse</a></li>
<li><a href="#how-to-limit-network-abuse-httrack-websi">How to limit network abuse HTTrack Website Copier FAQ (updated - DRAFT)</a></li>
</ul>
<h2>Start here</h2>
<ul>
<li><a href="guide.html">Interface guide</a></li>
<li><a href="faq.html">FAQ and troubleshooting</a></li>
<li><a href="abuse.html" aria-current="page" class="here">Using HTTrack responsibly</a></li>
</ul>
<h2>Command line</h2>
<ul>
<li><a href="cmdguide.html">Command-line guide</a></li>
<li><a href="httrack.man.html">Option reference</a></li>
<li><a href="filters.html">Filter syntax</a></li>
</ul>
<h2>Developers</h2>
<ul>
<li><a href="dev.html">Programming</a></li>
<li><a href="library.html">libhttrack API</a></li>
<li><a href="plug.html">Callbacks</a></li>
<li><a href="scripting.html">Scripting</a></li>
<li><a href="cache.html">Cache format</a></li>
<li><a href="changes.html">Change report format</a></li>
</ul>
<h2>More</h2>
<ul>
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
<li><a href="contact.html">Contact and credits</a></li>
<li><a href="index.html">All documentation</a></li>
</ul>
</nav>
<main id="main">
<!-- /doc-chrome:top -->
<h3 id="for-httrack-users">For HTTrack users:</h3>
<h3>For HTTrack users:</h3>
<ul>
<li><a href="#USERS">Advice & what <u>not</u> to do when you are using HTTrack</a></li>
</ul>
<br>
<h3 id="for-webmasters-having-problems-with-band">For webmasters having problems with bandwidth abuse / other abuses related to HTTrack:</h3>
<h3>For webmasters having problems with bandwidth abuse / other abuses related to HTTrack:</h3>
<ul>
<li><a href="#WEBMASTERS">Abuse FAQ for <u>webmasters</a></u></li>
</ul>
@@ -84,9 +121,9 @@
<h1>Advice & what <u>not</u> to do</h1>
<h2 align="center"><em>Advice & what <u>not</u> to do</em></h2>
<center><h3 id="please-follow-these-common-sense-rules-t">Please follow these <i>common sense</i> rules to avoid any network abuse</h3></center>
<center><h3>Please follow these <i>common sense</i> rules to avoid any network abuse</h3></center>
<br>
@@ -141,9 +178,9 @@ Downloading a site can overload it, if you have a fast pipe, or if you capture t
<h2 id="abuse-faq-for-webmasters" align="center"><em>Abuse FAQ for <u>webmasters</u></em></h2>
<h2 align="center"><em>Abuse FAQ for <u>webmasters</u></em></h2>
<center><h3 id="how-to-limit-network-abuse-httrack-websi">How to limit network abuse
<center><h3>How to limit network abuse
<br>
HTTrack Website Copier FAQ (updated - DRAFT)
</h3></center>
@@ -180,7 +217,7 @@ school or
shows. They might do that because they are connected through expensive modem connection,
or because they would like to consult pages while travelling, or archive sites that may be
removed
one day, make some data mining, compiling information (&quot;if only I could find this
one day, make some data mining, comiling information (&quot;if only I could find this
website I saw one day..&quot;). <br>
There are many good reasons to mirror websites, and this helps many good people.<br>
As a webmaster, you might be interested to use such tools, too: test broken links, move a
@@ -192,7 +229,7 @@ test the webserver response and performances, index it..<br>
Anyway, bandwidth abuse can be a problem. If your site is regularly &quot;clobbered&quot;
by evil downloaders, you have <br>
various solutions. You have radical solutions, and intermediate solutions. I strongly
recommend not to use<br>
recomment not to use<br>
radical solutions, because of the previous remarks (good people often mirror websites).<br>
<br>
In general, for all solutions,<br>
@@ -207,7 +244,7 @@ or, to be extreme: if you unplug the wire, there will be no bandwidth abuse<br>
Good: Will work with good people. Many good people just don't KNOW that they can slow down
a network.<br>
Bad: Will **only** work with good people<br>
How to do: Obvious - place a note, a warning, an article, a draw, a poem or whatever you
How to do: Obvious - place a note, a warning, an article, a draw, a poeme or whatever you
want<br>
<br>
</li><li>Use &quot;robots.txt&quot; file<br>
@@ -229,7 +266,7 @@ Good: Efficient<br>
Bad: Multiple users behind proxies will be slow down, not really easy to setup<br>
How to do: Depends on webserver. Might be done with low-level IP rules (QoS)<br>
<br>
</li><li>Prioritize small files, against large files<br>
</li><li>Priorize small files, against large files<br>
Good: Efficient if large files are the cause of abuse<br>
Bad: Not always efficient<br>
How to do: Depends on the webserver<br>
@@ -246,7 +283,7 @@ How to do: Use routine QoS (fair queuing), or webserver options<br>
<br>
</li><li>Use technical tricks (like javascript) to hide URLs<br>
Good: Efficient<br>
Bad: The most efficient tricks will also cause your website to be heavy, and not
Bad: The most efficient tricks will also cause your website to he heavy, and not
user-friendly (and therefore less attractive, even for surfing users). Remember: clients
or visitors might want to consult offline your website. Advanced users will also be still
able to note the URLs and catch them. Will not work on non-javascript browsers. It will
@@ -298,22 +335,22 @@ Example: Use things like
</li><li>Use technical tricks to temporarily ban IPs<br>
Good: Efficient<br>
Bad: Radical (your site will only be available online for all users), not easy to setup<br>
How to do: Create fake links with &quot;killing&quot; targets<br>
How to to: Create fake links with &quot;killing&quot; targets<br>
Example: Use things like &lt;a href=&quot;killme.cgi&quot;&gt;&lt;nothing&gt;&lt;/a&gt;
(again an example in php:)<br>
<tt>
<pre>
&lt;?php
// Add IP.
add_temp_firewall_rule($REMOTE_ADDR,"30s");
?&gt;
function add_temp_firewall_rule($addr) {
// The chain chhttp is flushed in a cron job to avoid ipchains overflow
system("/usr/bin/sudo -u root /sbin/ipchains -I 1 chhttp -p tcp -s ".$addr." --dport 80 -j REJECT");
syslog("user rejected due to too many copy attemps : ".$addr);
}
<tt>
<pre>
&lt;?php
// Add IP.
add_temp_firewall_rule($REMOTE_ADDR,"30s");
?&gt;
function add_temp_firewall_rule($addr) {
// The chain chhttp is flushed in a cron job to avoid ipchains overflow
system("/usr/bin/sudo -u root /sbin/ipchains -I 1 chhttp -p tcp -s ".$addr." --dport 80 -j REJECT");
syslog("user rejected due to too many copy attemps : ".$addr);
}
<pre>
</tt>
<br>
@@ -372,15 +409,15 @@ Example:<br>
&nbsp;&nbsp;&nbsp; FOS('mycompany.com','smith?subject=Hi, John','Click here to email me!')<br>
&nbsp;&nbsp;&nbsp; // --&gt;<br>
&nbsp;&nbsp;&nbsp; &lt;/script&gt;<br>
&nbsp;&nbsp;&nbsp; &lt;noscript&gt;<br>
&nbsp;&nbsp;&nbsp;&nbsp;smith at mycompany dot com<br>
&nbsp;&nbsp;&nbsp; &lt;/noscript&gt;<br>
&nbsp;&nbsp;&nbsp; &lt;noscript&gt;<br>
&nbsp;&nbsp;&nbsp;&nbsp;smith at mycompany dot com<br>
&nbsp;&nbsp;&nbsp; &lt;/noscript&gt;<br>
</tt>
<br>
</li><li>Another one is to create images of emails<br>
Good: Efficient, does not require javascript<br>
Bad: There is still the problem of the link (mailto:), images are bigger than text, and it can cause problems for blind people (a good solution is use an ALT attribute with the email written like "smith at mycompany dot com")<br>
How to do: Not so obvious if you do not want to create images by yourself<br>
How to do: Not so obvious of you do not want to create images by yourself<br>
Example: (php, Unix)<br>
<tt>
@@ -454,7 +491,7 @@ echo <br>
</li><li>You can also create temporary email aliases, each week, for all users<br>
Good: Efficient, and you can give your real email in your reply-to address<br>
Bad: Temporary emails<br>
How to do: Not so hard to do<br>
How to do: Not so hard todo<br>
Example: (script &amp; php, Unix)<br>
<tt>
@@ -529,14 +566,26 @@ And then, put the email address in your pages through:
<br>
<!-- doc-chrome:bottom -->
</main>
</div>
<!-- ==================== Start epilogue ==================== -->
</td>
</tr>
</table>
</td>
</tr>
</table>
</td>
</tr>
</table>
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
<footer>&copy; 1998-2026 Xavier Roche &amp; other contributors - Web Design: Leto Kauler.</footer>
<!-- /doc-chrome:bottom -->
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
<tr>
<td id="footer"><small>&copy; 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
</tr>
</table>
</body>
</html>

159
html/addurl.html
View File

@@ -1,10 +1,155 @@
<!DOCTYPE html>
<html lang="en">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<meta charset="utf-8">
<title>Moved to the HTTrack interface guide</title>
<link rel="canonical" href="guide.html#step-address">
<meta http-equiv="refresh" content="0; url=guide.html#step-address">
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
<title>HTTrack Website Copier - Offline Browser</title>
<style type="text/css">
<!--
body {
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
background: #77b;
}
body, td {
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
}
#subTitle {
background: #000; color: #fff; padding: 4px; font-weight: bold;
}
#siteNavigation a, #siteNavigation .current {
font-weight: bold; color: #448;
}
#siteNavigation a:link { text-decoration: none; }
#siteNavigation a:visited { text-decoration: none; }
#siteNavigation .current { background-color: #ccd; }
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
a:link { text-decoration: underline; color: #00f; }
a:visited { text-decoration: underline; color: #000; }
a:hover { text-decoration: underline; color: #c00; }
a:active { text-decoration: underline; }
#pageContent {
clear: both;
border-bottom: 6px solid #000;
padding: 10px; padding-top: 20px;
line-height: 1.65em;
background-image: url(images/bg_rings.gif);
background-repeat: no-repeat;
background-position: top right;
}
#pageContent, #siteNavigation {
background-color: #ccd;
}
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
h1 { margin: 0; font-weight: bold; font-size: 2em; }
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
.blak { background-color: #000; }
.hide { display: none; }
.tableWidth { min-width: 400px; }
.tblRegular { border-collapse: collapse; }
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
.tblNoBorder td { border: 0; }
// -->
</style>
</head>
<body><p>Adding a URL is now part of <a href="guide.html#step-address">the HTTrack interface guide</a>.</p></body>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr>
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
<tr>
<td id="subTitle">Open Source offline browser</td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr class="blak">
<td>
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
<tr>
<td colspan="6">
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
<tr>
<td id="pageContent">
<!-- ==================== End prologue ==================== -->
<h2 align="center"><em>Add a URL</em></h2>
<br>
<ol>
<li>Enter a typical Web address</li>
<br><small>Just type in your address in the field</small><br><br>
<center><img src="img/addurl1.gif" border="0"></center>
<br><br>OR<br><br>
<li>Enter a Web address with authentication</li>
<br><small>Useful when you need basic authentication to watch the Web page</small><br><br>
<center><img src="img/addurl2.gif" border="0"></center>
<br><br>OR<br><br>
<li>Capture a link from your Web browser to HTTrack</li>
<br><small>Use this tool only for form-based pages (pages delivered after submiting a form) that need some analysis</small><br><br>
<center><img src="img/addurl3.gif" border="0"></center>
<br>
<br>
<small>Set, as explained, your Web browser proxy preferences to the values indicated : set the proxy's address, and the proxy's port,
then click on the button or link as you usually do in your Web browser.
<br>The temporary proxy, installed by HTTrack, will then
capture the link and display a confirmation page.
<br><br>
</small>
<center><img src="img/addurl4.gif" border="0"></center>
<br>
<center><img src="img/addurl5.gif" border="0"></center>
</ol>
<br><br><br><br>
<p align="right">Back to <a href="index.html">Home</a></p>
<!-- ==================== Start epilogue ==================== -->
</td>
</tr>
</table>
</td>
</tr>
</table>
</td>
</tr>
</table>
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
<tr>
<td id="footer"><small>&copy; 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
</tr>
</table>
</body>
</html>

11
html/android.html
View File

@@ -1,11 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Moved to the HTTrack interface guide</title>
<link rel="canonical" href="guide.html">
<meta http-equiv="refresh" content="0; url=guide.html#droid/">
</head>
<body><p>This page is now part of
<a href="guide.html#droid/">the HTTrack interface guide</a>.</p></body>
</html>

503
html/cache.html
View File

@@ -1,238 +1,293 @@
<!DOCTYPE html>
<html lang="en">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<!-- doc-chrome:head -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="The format of the HTTrack cache in hts-cache, and how to read it.">
<link rel="stylesheet" href="doc.css">
<script src="doc.js" defer></script>
<!-- /doc-chrome:head -->
<title>HTTrack cache format</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
<title>HTTrack Website Copier - Cache format specification</title>
<style type="text/css">
<!--
body {
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
background: #77b;
}
body, td {
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
}
#subTitle {
background: #000; color: #fff; padding: 4px; font-weight: bold;
}
#siteNavigation a, #siteNavigation .current {
font-weight: bold; color: #448;
}
#siteNavigation a:link { text-decoration: none; }
#siteNavigation a:visited { text-decoration: none; }
#siteNavigation .current { background-color: #ccd; }
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
a:link { text-decoration: underline; color: #00f; }
a:visited { text-decoration: underline; color: #000; }
a:hover { text-decoration: underline; color: #c00; }
a:active { text-decoration: underline; }
#pageContent {
clear: both;
border-bottom: 6px solid #000;
padding: 10px; padding-top: 20px;
line-height: 1.65em;
background-image: url(images/bg_rings.gif);
background-repeat: no-repeat;
background-position: top right;
}
#pageContent, #siteNavigation {
background-color: #ccd;
}
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
h1 { margin: 0; font-weight: bold; font-size: 2em; }
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
.blak { background-color: #000; }
.hide { display: none; }
.tableWidth { min-width: 400px; }
.tblRegular { border-collapse: collapse; }
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
.tblNoBorder td { border: 0; }
// -->
</style>
</head>
<body>
<!-- doc-chrome:top -->
<a class="skip" href="#main">Skip to content</a>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr>
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
<tr>
<td id="subTitle">Open Source offline browser</td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr class="blak">
<td>
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
<tr>
<td colspan="6">
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
<tr>
<td id="pageContent">
<!-- ==================== End prologue ==================== -->
<header class="masthead">
<img src="images/wordmark.svg" width="400" height="36" alt="HTTrack Website Copier">
<div class="tagline">Open Source offline browser</div>
</header>
<div class="wrap">
<nav class="toc" aria-label="Documentation">
<h2>Start here</h2>
<ul>
<li><a href="guide.html">Interface guide</a></li>
<li><a href="faq.html">FAQ and troubleshooting</a></li>
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
</ul>
<h2>Command line</h2>
<ul>
<li><a href="cmdguide.html">Command-line guide</a></li>
<li><a href="httrack.man.html">Option reference</a></li>
<li><a href="filters.html">Filter syntax</a></li>
</ul>
<h2>Developers</h2>
<ul>
<li><a href="dev.html">Programming</a></li>
<li><a href="library.html">libhttrack API</a></li>
<li><a href="plug.html">Callbacks</a></li>
<li><a href="scripting.html">Scripting</a></li>
<li><a href="cache.html" aria-current="page" class="here">Cache format</a></li>
<li><a href="changes.html">Change report format</a></li>
</ul>
<h2>More</h2>
<ul>
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
<li><a href="contact.html">Contact and credits</a></li>
<li><a href="index.html">All documentation</a></li>
</ul>
</nav>
<main id="main">
<!-- /doc-chrome:top -->
<h1>Cache format specification</h1>
<h2 align="center"><em>Cache format specification</em></h2>
<br />
For updating purpose, HTTrack stores original (untouched) HTML data,
references to downloaded files, and other meta-data (especially parts of the HTTP headers) in a cache,
located in the hts-cache directory. Because local html pages are always modified to "fit" the local
filesystem structure, and because meta-data such as the last-Modified date and Etag can not be stored
with the associated files, the cache is absolutely mandatory for reprocessing (update/continue) phases.
For updating purpose, HTTrack stores original (untouched) HTML data,
references to downloaded files, and other meta-data (especially parts of the HTTP headers) in a cache,
located in the hts-cache directory. Because local html pages are always modified to "fit" the local
filesystem structure, and because meta-data such as the last-Modified date and Etag can not be stored
with the associated files, the cache is absolutely mandatory for reprocessing (update/continue) phases.
<br /><br />
<h3>The (new) cache.zip format</h3>
The 3.31 release of HTTrack introduces a new cache format, more extensible and efficient than the previous one (ndx/dat format).
The main advantages of this cache are:
<ul>
<li>One single file for a complete website cache archive</li>
<li>Standard <a href="http://www.pkware.com/products/enterprise/white_papers/appnote.txt" target="_new">ZIP</a> format, that can be easily reused on most platforms and languages</li>
<li>Compressed data with the efficient and opened <a href="http://www.gzip.org/zlib/" target="_new">zlib</a> format</li>
</ul>
The cache is made of ZIP files entries ; with one ZIP file entry per fetched URL (successfully or not - errors are also stored).<br />
For each entry:
<ul>
<li>The ZIP file name is the original URL [<small><a href="#orig">see notes below</a></small>]</li>
<li>The ZIP file contents, <b>if available</b>, is the original (compressed, using the deflate algorythm) data</li>
<li>The ZIP file extra field (in the local file header) contains a list of meta-fields, very similar to the <a href="http://www.ietf.org/rfc/rfc2616.txt?number=2616" target="new_">HTTP</a> headers fields. See also <a href="http://www.ietf.org/rfc/rfc2396.txt?number=2396" target="new_">RFC</a>.</li><br />
<li>The ZIP file timestamp follows the "Last-Modified-Since" field given for this URL, if any</li>
</ul>
Example of cache file:
<br />
<pre>
$ unzip -l hts-cache/new.zip
Archive: hts-cache/new.zip
HTTrack Website Copier/3.31-ALPHA-4 mirror complete in 3 seconds : 5 links scanned,
3 files written (16109 bytes overall) [17690 bytes received at 5896 bytes/sec]
(1 errors, 0 warnings, 0 messages)
Length Date Time Name
-------- ---- ---- ----
94 07-18-03 08:59 http://www.httrack.com/robots.txt
9866 01-17-04 01:09 http://www.httrack.com/html/cache.html
0 05-11-03 13:31 http://www.httrack.com/html/images/bg_rings.gif
207 01-19-04 05:49 http://www.httrack.com/html/fade.gif
0 05-11-03 13:31 http://www.httrack.com/html/images/header_title_4.gif
-------- -------
10167 5 files
</pre>
Example of cache file meta-data:
<br />
<pre>
HTTP/1.1 200 OK
X-In-Cache: 1
X-StatusCode: 200
X-StatusMessage: OK
X-Size: 94
Content-Type: text/plain
Last-Modified: Fri, 18 Jul 2003 08:59:11 GMT
Etag: "40ebb5-5e-3f17b6df"
X-Addr: www.httrack.com
X-Fil: /robots.txt
</pre>
There are also specific issues regarding this format:
<ul>
<li>The data in the central directory (such as CD extra field, and CD comments) are not used</li>
<li>The ZIP archive is allowed to contains more than 2^16 files (65535) ; in such case the total number of entries in the 32-bit central directory is 65536 (0xffff), but the presence of the 64-bit central directory is not mandatory</li>
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory must be present <b>(not currently supported)</b></li>
</ul>
<br />
<b>Meta-data stored in the "extra field" of the local file headers</b><br />
The extra field is composed of text data, and this text data is composed of distinct lines of headers.
The end of text, <b>or</b> a double CR/LF, mark the end of this zone.
This method allows you to optionally store original HTTP headers just after the "meta-data" headers for informational use.<br />
<br />
<b>The status line (the first headers line)</b><br />
Status-Line = HTTP-Version SP Status-Code SP X-Reason-Phrase CRLF<br />
<br />
<b>Other lines:</b><br />
<br />
<b>Specific fields:</b><br />
<ul>
<li>X-In-Cache</li><br>
Indicates if the data are present (value=1) in the cache (that is, as ZIP data), or in an external file (value=0).
This field MUST be the first field.
<li>X-StatusCode</li><br>
The modified (by httrack) status code after processing. 304 error codes ("Not modified"), for example, are transformed into "200" codes after processing.
<li>X-StatusMessage</li><br>
The modified (by httrack) status message.
<li>X-Size</li><br>
The stored (either in cache, or in an external file) data size.
<li>X-Charset</li><br>
The original charset.
<li>X-Addr</li><br>
The original URL address part.
<li>X-Fil</li><br>
The original URL path part.
<li>X-Save</li><br>
The local filename, depending on user's "build structure" preferences.
</ul>
<br />
<b>Standard (RFC 2616) "useful" fields:</b><br />
<ul>
<li>Content-Type</li>
<li>Last-Modified</li>
<li>Etag</li>
<li>Location</li>
<li>Content-Disposition</li>
</ul>
<br />
<b>Specific fields in "BNF-like" grammar:</b><br />
<pre>
X-In-Cache = "X-In-Cache" ":" 1*DIGIT
X-StatusCode = "X-StatusCode" ":" 1*DIGIT
X-StatusMessage = "X-StatusMessage" ":" *&lt;TEXT, excluding CR, LF&gt;
X-Size = "X-Size" ":" 1*DIGIT
X-Charset = "X-Charset" ":" value
X-Addr = "X-Addr" ":" scheme ":" "//" authority
X-Fil = "X-Fil" ":" rel_path
X-Save = "X-Save" ":" rel_path
</pre>
<b>RFC standard fields:</b><br />
<pre>
Content-Type = "Content-Type" ":" media-type
Last-Modified = "Last-Modified" ":" HTTP-date
Etag = "ETag" ":" entity-tag
Location = "Location" ":" absoluteURI
Content-Disposition = "Content-Disposition" ":" disposition-type *( ";" disposition-parm )
</pre>
<br />
And, for your information,
<pre>
X-Reason-Phrase = *&lt;TEXT, with a maximum of 32 characters, and excluding CR, LF&gt;
</pre>
<a name="orig">Note: Because the URLs may have an unexpected format, especially with double "/" inside, and other reserved characters ("?", "&" ..),
various ZIP uncompressors can potentially have troubles accessing or decompressing the data.
Libraries should generally handle this peculiar format, however.
</a>
<br /><br />
<h3 id="the-new-cache-zip-format">The (new) cache.zip format</h3>
<!-- ==================== Start epilogue ==================== -->
</td>
</tr>
</table>
</td>
</tr>
</table>
</td>
</tr>
</table>
The 3.31 release of HTTrack introduces a new cache format, more extensible and efficient than the previous one (ndx/dat format).
The main advantages of this cache are:
<ul>
<li>One single file for a complete website cache archive</li>
<li>Standard <a href="http://www.pkware.com/products/enterprise/white_papers/appnote.txt" target="_new">ZIP</a> format, that can be easily reused on most platforms and languages</li>
<li>Compressed data with the efficient and opened <a href="http://www.gzip.org/zlib/" target="_new">zlib</a> format</li>
</ul>
The cache is made of ZIP files entries ; with one ZIP file entry per fetched URL (successfully or not - errors are also stored).<br />
For each entry:
<ul>
<li>The ZIP file name is the original URL [<small><a href="#orig">see notes below</a></small>]</li>
<li>The ZIP file contents, <b>if available</b>, is the original (compressed, using the deflate algorythm) data</li>
<li>The ZIP file extra field (in the local file header) contains a list of meta-fields, very similar to the <a href="http://www.ietf.org/rfc/rfc2616.txt?number=2616" target="new_">HTTP</a> headers fields. See also <a href="http://www.ietf.org/rfc/rfc2396.txt?number=2396" target="new_">RFC</a>.</li><br />
<li>The ZIP file timestamp follows the "Last-Modified-Since" field given for this URL, if any</li>
</ul>
Example of cache file:
<br />
<pre>
$ unzip -l hts-cache/new.zip
Archive: hts-cache/new.zip
HTTrack Website Copier/3.31-ALPHA-4 mirror complete in 3 seconds : 5 links scanned,
3 files written (16109 bytes overall) [17690 bytes received at 5896 bytes/sec]
(1 errors, 0 warnings, 0 messages)
Length Date Time Name
-------- ---- ---- ----
94 07-18-03 08:59 http://www.httrack.com/robots.txt
9866 01-17-04 01:09 http://www.httrack.com/html/cache.html
0 05-11-03 13:31 http://www.httrack.com/html/images/bg_rings.gif
207 01-19-04 05:49 http://www.httrack.com/html/fade.gif
0 05-11-03 13:31 http://www.httrack.com/html/images/header_title_4.gif
-------- -------
10167 5 files
</pre>
Example of cache file meta-data:
<br />
<pre>
HTTP/1.1 200 OK
X-In-Cache: 1
X-StatusCode: 200
X-StatusMessage: OK
X-Size: 94
Content-Type: text/plain
Last-Modified: Fri, 18 Jul 2003 08:59:11 GMT
Etag: "40ebb5-5e-3f17b6df"
X-Addr: www.httrack.com
X-Fil: /robots.txt
</pre>
There are also specific issues regarding this format:
<ul>
<li>The data in the central directory (such as CD extra field, and CD comments) are not used</li>
<li>The ZIP archive is allowed to contains more than 2^16 files (65535) ; in such case the total number of entries in the 32-bit central directory is 65536 (0xffff), but the presence of the 64-bit central directory is not mandatory</li>
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory is emitted automatically (a single stored entry of 4GiB or more is not supported)</li>
</ul>
<br />
<b>Meta-data stored in the "extra field" of the local file headers</b><br />
The extra field is composed of text data, and this text data is composed of distinct lines of headers.
The end of text, <b>or</b> a double CR/LF, mark the end of this zone.
This method allows you to optionally store original HTTP headers just after the "meta-data" headers for informational use.<br />
<br />
<b>The status line (the first headers line)</b><br />
Status-Line = HTTP-Version SP Status-Code SP X-Reason-Phrase CRLF<br />
<br />
<b>Other lines:</b><br />
<br />
<b>Specific fields:</b><br />
<ul>
<li>X-In-Cache</li><br>
Indicates if the data are present (value=1) in the cache (that is, as ZIP data), or in an external file (value=0).
This field MUST be the first field.
<li>X-StatusCode</li><br>
The modified (by httrack) status code after processing. 304 error codes ("Not modified"), for example, are transformed into "200" codes after processing.
<li>X-StatusMessage</li><br>
The modified (by httrack) status message.
<li>X-Size</li><br>
The stored (either in cache, or in an external file) data size.
<li>X-Charset</li><br>
The original charset.
<li>X-Addr</li><br>
The original URL address part.
<li>X-Fil</li><br>
The original URL path part.
<li>X-Save</li><br>
The local filename, depending on user's "build structure" preferences.
</ul>
<br />
<b>Standard (RFC 2616) "useful" fields:</b><br />
<ul>
<li>Content-Type</li>
<li>Last-Modified</li>
<li>Etag</li>
<li>Location</li>
<li>Content-Disposition</li>
</ul>
<br />
<b>Specific fields in "BNF-like" grammar:</b><br />
<pre>
X-In-Cache = "X-In-Cache" ":" 1*DIGIT
X-StatusCode = "X-StatusCode" ":" 1*DIGIT
X-StatusMessage = "X-StatusMessage" ":" *&lt;TEXT, excluding CR, LF&gt;
X-Size = "X-Size" ":" 1*DIGIT
X-Charset = "X-Charset" ":" value
X-Addr = "X-Addr" ":" scheme ":" "//" authority
X-Fil = "X-Fil" ":" rel_path
X-Save = "X-Save" ":" rel_path
</pre>
<b>RFC standard fields:</b><br />
<pre>
Content-Type = "Content-Type" ":" media-type
Last-Modified = "Last-Modified" ":" HTTP-date
Etag = "ETag" ":" entity-tag
Location = "Location" ":" absoluteURI
Content-Disposition = "Content-Disposition" ":" disposition-type *( ";" disposition-parm )
</pre>
<br />
And, for your information,
<pre>
X-Reason-Phrase = *&lt;TEXT, with a maximum of 32 characters, and excluding CR, LF&gt;
</pre>
<a name="orig">Note: Because the URLs may have an unexpected format, especially with double "/" inside, and other reserved characters ("?", "&" ..),
various ZIP uncompressors can potentially have troubles accessing or decompressing the data.
Libraries should generally handle this peculiar format, however.
</a>
<br /><br />
<!-- doc-chrome:bottom -->
</main>
</div>
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
<footer>&copy; 1998-2026 Xavier Roche &amp; other contributors - Web Design: Leto Kauler.</footer>
<!-- /doc-chrome:bottom -->
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
<tr>
<td id="footer"><small>&copy; 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
</tr>
</table>
</body>
</html>

217
html/changes.html
View File

@@ -1,217 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<!-- doc-chrome:head -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="The change report HTTrack writes after an update, field by field.">
<link rel="stylesheet" href="doc.css">
<script src="doc.js" defer></script>
<!-- /doc-chrome:head -->
<title>HTTrack change report format</title>
</head>
<body>
<!-- doc-chrome:top -->
<a class="skip" href="#main">Skip to content</a>
<header class="masthead">
<img src="images/wordmark.svg" width="400" height="36" alt="HTTrack Website Copier">
<div class="tagline">Open Source offline browser</div>
</header>
<div class="wrap">
<nav class="toc" aria-label="Documentation">
<h2>On this page</h2>
<ul>
<li><a href="#what-changed-means">What "changed" means</a></li>
<li><a href="#with-the-cache-off">With the cache off</a></li>
<li><a href="#fields">Fields</a></li>
<li><a href="#encoding">Encoding</a></li>
<li><a href="#example">Example</a></li>
<li><a href="#notes">Notes</a></li>
</ul>
<h2>Start here</h2>
<ul>
<li><a href="guide.html">Interface guide</a></li>
<li><a href="faq.html">FAQ and troubleshooting</a></li>
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
</ul>
<h2>Command line</h2>
<ul>
<li><a href="cmdguide.html">Command-line guide</a></li>
<li><a href="httrack.man.html">Option reference</a></li>
<li><a href="filters.html">Filter syntax</a></li>
</ul>
<h2>Developers</h2>
<ul>
<li><a href="dev.html">Programming</a></li>
<li><a href="library.html">libhttrack API</a></li>
<li><a href="plug.html">Callbacks</a></li>
<li><a href="scripting.html">Scripting</a></li>
<li><a href="cache.html">Cache format</a></li>
<li><a href="changes.html" aria-current="page" class="here">Change report format</a></li>
</ul>
<h2>More</h2>
<ul>
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
<li><a href="contact.html">Contact and credits</a></li>
<li><a href="index.html">All documentation</a></li>
</ul>
</nav>
<main id="main">
<!-- /doc-chrome:top -->
<h1>Change report format specification</h1>
<br />
Run with <tt>--changes</tt> (<tt>-%d</tt>), HTTrack writes <tt>hts-changes.json</tt>
in the project directory, next to <tt>hts-log.txt</tt>, describing what the crawl
left new, changed, unchanged and gone compared to the previous mirror. The file is
rewritten from scratch at the end of every run, and the log carries a one-line
summary of the same counts.
<br /><br />
<h3 id="what-changed-means">What "changed" means</h3>
A resource is changed when its bytes differ, not when the server merely re-sent
it. HTTrack compares the payload it just received against the copy the previous
run left behind: for pages it parses, the previous payload comes from the cache
(the file on disk carries the mirror footer and its crawl date, so its bytes
differ on every run); for everything else, the mirrored file is the payload
verbatim and is compared directly.
<br /><br />
Where no digest can be taken on either side, because the cache is disabled or
the previous copy is gone, the report falls back to the transfer signal, and a
server that answers 200 rather than 304 reads as changed. Keeping the cache on
(the default) is what makes the report precise.
<br /><br />
<h3 id="with-the-cache-off">With the cache off</h3>
<tt>--cache=0</tt> costs the report more than the digest of a parsed page. The
mirror's file index (<tt>hts-cache/new.lst</tt>) is what records which files a
run produced, so without it there is no previous mirror to subtract from: nothing
is reported <tt>gone</tt>, and whether the run is a first crawl cannot be decided
at all, which <tt>first_crawl</tt> states as <tt>null</tt> rather than guess. What
is on disk is still compared byte for byte, so the other three lists stay
meaningful, except for the pages HTTrack parses: those have no cached payload to
compare against and fall back to the transfer signal.
<br /><br />
<h3 id="fields">Fields</h3>
<ul>
<li><tt>schema</tt>: format version, currently <tt>1</tt>. It is bumped only
on an incompatible change; new fields may appear without one.</li>
<li><tt>generator</tt>: the HTTrack build that wrote the file.</li>
<li><tt>date</tt>: when the report was written, UTC, <tt>YYYY-MM-DDThh:mm:ssZ</tt>.</li>
<li><tt>first_crawl</tt>: true when no index of a previous mirror
(<tt>hts-cache/old.lst</tt>) was found, so there was nothing to compare against and
everything is listed as new. Null when the run kept no index at all and the
question cannot be answered (see above).</li>
<li><tt>partial</tt>: true when the report ran out of memory and lists only
part of the mirror.</li>
<li><tt>purged</tt>: true when <tt>--purge-old</tt> was in effect, so the
files under <tt>gone</tt> were also deleted from disk.</li>
<li><tt>counts</tt>: the size of each of the four lists.</li>
<li><tt>new</tt>, <tt>changed</tt>, <tt>unchanged</tt>, <tt>gone</tt>: the
lists themselves. Every mirrored file appears in exactly one of them.</li>
</ul>
Each entry is an object:
<ul>
<li><tt>url</tt>: the absolute URL the file came from. Empty under
<tt>gone</tt>: deletions are computed from the mirror's file index, which records
paths, not URLs.</li>
<li><tt>file</tt>: the path relative to the mirror root, with forward
slashes. This is the entry's identity: a URL and a redirect that resolve to the
same local file are one entry, not two.</li>
<li><tt>size</tt>: the mirrored file's size in bytes, absent when the file
is not on disk.</li>
<li><tt>previous_size</tt>: under <tt>changed</tt> only, the size of the
copy the previous run left.</li>
</ul>
<br />
<h3 id="encoding">Encoding</h3>
The file is JSON, UTF-8. URLs and local paths reach HTTrack as raw bytes and are
not guaranteed to be valid UTF-8; any byte sequence that is not becomes
U+FFFD (<tt>\ufffd</tt>), so the file always parses. Compare on <tt>file</tt>
rather than on <tt>url</tt> when a mirror is known to carry legacy-charset URLs.
<br /><br />
<h3 id="example">Example</h3>
<pre>
{
"schema": 1,
"generator": "HTTrack Website Copier/3.49-14",
"date": "2026-07-26T15:29:03Z",
"first_crawl": false,
"partial": false,
"purged": true,
"counts": { "new": 1, "changed": 1, "unchanged": 1, "gone": 1 },
"new": [
{ "url": "http://example.com/d.html", "file": "example.com/d.html", "size": 280 }
],
"changed": [
{ "url": "http://example.com/a.html", "file": "example.com/a.html", "size": 281, "previous_size": 273 }
],
"unchanged": [
{ "url": "http://example.com/b.html", "file": "example.com/b.html", "size": 277 }
],
"gone": [
{ "url": "", "file": "example.com/c.html" }
]
}
</pre>
<br /><br />
<h3 id="notes">Notes</h3>
<ul>
<li>A file listed under <tt>gone</tt> is only deleted when <tt>--purge-old</tt> is
on. Left in place it drops out of the mirror's index, so it is reported once and
not again.</li>
<li>A resource whose local file name changed since the previous mirror (a new
MIME type, say) is reported as <tt>new</tt> under its new name; the old name is
reported as <tt>gone</tt> only if the file is still on disk. The two entries are
not paired.</li>
<li>A resource this run tried and failed to transfer also drops out of the
mirror's index, but its previous copy is untouched, so it is reported
<tt>unchanged</tt>. Under <tt>--purge-old</tt> that copy is deleted anyway, and
the report says <tt>gone</tt> to match.</li>
<li>A run that transfers no data at all is rolled back: HTTrack restores the
previous cache generation and leaves the previous report in place, so a lost
connection does not overwrite a good report with an empty one.</li>
<li>Content diffs, and keeping the previous copy of a changed page, are out of
scope: both change what a mirror directory contains.</li>
</ul>
<br /><br />
<!-- doc-chrome:bottom -->
</main>
</div>
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
<footer>&copy; 1998-2026 Xavier Roche &amp; other contributors - Web Design: Leto Kauler.</footer>
<!-- /doc-chrome:bottom -->
</body>
</html>

159
html/cmddoc.html
View File

@@ -1,10 +1,155 @@
<!DOCTYPE html>
<html lang="en">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<meta charset="utf-8">
<title>Moved to the command-line guide</title>
<link rel="canonical" href="cmdguide.html">
<meta http-equiv="refresh" content="0; url=cmdguide.html">
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
<title>HTTrack Website Copier - Offline Browser</title>
<style type="text/css">
<!--
body {
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
background: #77b;
}
body, td {
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
}
#subTitle {
background: #000; color: #fff; padding: 4px; font-weight: bold;
}
#siteNavigation a, #siteNavigation .current {
font-weight: bold; color: #448;
}
#siteNavigation a:link { text-decoration: none; }
#siteNavigation a:visited { text-decoration: none; }
#siteNavigation .current { background-color: #ccd; }
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
a:link { text-decoration: underline; color: #00f; }
a:visited { text-decoration: underline; color: #000; }
a:hover { text-decoration: underline; color: #c00; }
a:active { text-decoration: underline; }
#pageContent {
clear: both;
border-bottom: 6px solid #000;
padding: 10px; padding-top: 20px;
line-height: 1.65em;
background-image: url(images/bg_rings.gif);
background-repeat: no-repeat;
background-position: top right;
}
#pageContent, #siteNavigation {
background-color: #ccd;
}
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
h1 { margin: 0; font-weight: bold; font-size: 2em; }
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
.blak { background-color: #000; }
.hide { display: none; }
.tableWidth { min-width: 400px; }
.tblRegular { border-collapse: collapse; }
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
.tblNoBorder td { border: 0; }
// -->
</style>
</head>
<body><p>This page is now part of <a href="cmdguide.html">the HTTrack command-line guide</a>.</p></body>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr>
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
<tr>
<td id="subTitle">Open Source offline browser</td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr class="blak">
<td>
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
<tr>
<td colspan="6">
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
<tr>
<td id="pageContent">
<!-- ==================== End prologue ==================== -->
<h2 align="center"><em>Command-Line Documentation</em></h2>
<br>
The command-line version
<ul>
<li><a href="options.html">Command line Options</a></li>
<br>List of all powerful command line options<br><br>
<li>How to use httrack command-line version:</li>
<ul>
<li>Open a shell window</li>
<br>
<br>
<li>Type in <tt>httrack</tt> (or the complete path to the httrack executable)</li>
<br><small><tt>httrack</tt></small>
<br>
<br>
<li>Add the URLs, separated by a blank space</li>
<br><small><tt>httrack www.example.com/foo/</tt></small>
<br>
<br>
<li>If you need, add some options (see the <a href="options.html">option list</a>)</li>
<br><small><tt>httrack www.example.com/foo/ -O "/webs" -N4 -P proxy.myhost.com:3128</tt></small>
<br>
<br>
<li>Launch the command line, and wait until the mirror is finishing</li>
<br><small>You can (especially on the Unix release) press ^C to stop the mirror or put httrack in background</small>
<br>
<br>
</ul>
</ul>
<!-- ==================== Start epilogue ==================== -->
</td>
</tr>
</table>
</td>
</tr>
</table>
</td>
</tr>
</table>
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
<tr>
<td id="footer"><small>&copy; 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
</tr>
</table>
</body>
</html>

511
html/cmdguide.html
View File

@@ -1,511 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<!-- doc-chrome:head -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="A task-oriented guide to the httrack command line: scope, filters, limits, logins, proxies, updates and recipes.">
<link rel="stylesheet" href="doc.css">
<script src="doc.js" defer></script>
<!-- /doc-chrome:head -->
<title>HTTrack command-line guide</title>
</head>
<body>
<!-- doc-chrome:top -->
<a class="skip" href="#main">Skip to content</a>
<header class="masthead">
<img src="images/wordmark.svg" width="400" height="36" alt="HTTrack Website Copier">
<div class="tagline">Open Source offline browser</div>
</header>
<div class="wrap">
<nav class="toc" aria-label="Documentation">
<h2>On this page</h2>
<ul>
<li><a href="#quickstart">1. Quick start</a></li>
<li><a href="#scope">2. Scope: how far the crawl reaches</a></li>
<li><a href="#filters">3. Filters and scan rules</a></li>
<li><a href="#limits">4. Limits and politeness</a></li>
<li><a href="#names">5. File names and types</a></li>
<li><a href="#links">6. Links and page building</a></li>
<li><a href="#identity">7. Identity, cookies and login</a></li>
<li><a href="#proxy">8. Proxy and network</a></li>
<li><a href="#update">9. Update and cache</a></li>
<li><a href="#experts">10. Experts and scripting</a></li>
<li><a href="#recipes">11. Recipes</a></li>
</ul>
<h2>Start here</h2>
<ul>
<li><a href="guide.html">Interface guide</a></li>
<li><a href="faq.html">FAQ and troubleshooting</a></li>
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
</ul>
<h2>Command line</h2>
<ul>
<li><a href="cmdguide.html" aria-current="page" class="here">Command-line guide</a></li>
<li><a href="httrack.man.html">Option reference</a></li>
<li><a href="filters.html">Filter syntax</a></li>
</ul>
<h2>Developers</h2>
<ul>
<li><a href="dev.html">Programming</a></li>
<li><a href="library.html">libhttrack API</a></li>
<li><a href="plug.html">Callbacks</a></li>
<li><a href="scripting.html">Scripting</a></li>
<li><a href="cache.html">Cache format</a></li>
<li><a href="changes.html">Change report format</a></li>
</ul>
<h2>More</h2>
<ul>
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
<li><a href="contact.html">Contact and credits</a></li>
<li><a href="index.html">All documentation</a></li>
</ul>
</nav>
<main id="main">
<!-- /doc-chrome:top -->
<h1>Command-Line Guide</h1>
<p>This is a task-oriented guide to the <tt>httrack</tt> command line: how to do the
things people actually ask for, and the handful of defaults that surprise
newcomers. It sits on top of the
<a href="httrack.man.html">generated manual page</a>, which lists every option in
full. When you want the exhaustive detail for a flag, that page is the reference;
this one is the map.</p>
<p>Two habits before anything else. First, HTTrack has its own options: they are not
wget or curl flags, so reach for the tables here rather than guessing. Second,
when a mirror does something you did not expect, the answer is almost always in
the log. Every project writes <tt>hts-log.txt</tt> (and <tt>hts-err.txt</tt>) into
its output directory, and those files name every URL that was refused, redirected,
or filtered out. Read them first.</p>
<h3 id="quickstart">1. Quick start</h3>
<p>A mirror is one command: a start URL and an output directory.</p>
<p><tt>httrack https://example.com/ --path mydir</tt></p>
<p>With no other options HTTrack mirrors that site, stays on the same host, follows
links to any depth, rebuilds them to browse offline, and stores everything under
<tt>mydir</tt>. The same directory also holds the log files and the
<tt>hts-cache/</tt> folder that makes a later update or resume possible.</p>
<p>Two defaults are worth knowing up front, because both catch people out:</p>
<ul>
<li>HTTrack throttles itself to about <b>100 KB/s</b> even when you pass no rate
option. If a mirror feels slow, that is why. See
<a href="#limits">Limits</a> for how to lift it.</li>
<li>The download proceeds as a well-behaved robot: it identifies itself as
<tt>HTTrack</tt>, obeys <tt>robots.txt</tt>, and sends a Referer with each
request. A site that blocks that behavior needs the levers in
<a href="#identity">Identity</a>, not brute force.</li>
</ul>
<h3 id="scope">2. Scope: how far the crawl reaches</h3>
<p>Scope decides which links HTTrack is even willing to follow, before any filter
you write. Get this right and most "it downloaded too much" or "it only grabbed
the index" problems disappear.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--depth (-r)</tt></td><td>Maximum link depth. <b>The start page is level 1</b>, so one level of links out is <tt>-r2</tt>, not <tt>-r1</tt>.</td></tr>
<tr><td><tt>--stay-on-same-address (-a), --stay-on-same-domain (-d), --stay-on-same-tld (-l), --go-everywhere (-e)</tt></td><td>How far off the starting host the crawl may travel: same address (host), same principal domain, same top-level domain (for example .com), or everywhere. The default keeps you on the starting host.</td></tr>
<tr><td><tt>--can-go-down (-D), --can-go-up (-U), --stay-on-same-dir (-S), --can-go-up-and-down (-B)</tt></td><td>Directory travel: down into subdirectories only, up to parent directories only, stay in the same directory, or both up and down.</td></tr>
<tr><td><tt>--near (-n)</tt></td><td>Also fetch non-HTML files "near" a followed link, such as an image linked from a page you kept but hosted elsewhere.</td></tr>
<tr><td><tt>--ext-depth (-%e)</tt></td><td>How many levels of external links to follow once the crawl leaves your scope (default 0).</td></tr>
<tr><td><tt>--test (-t)</tt></td><td>Also HEAD-test links that fall outside the scope, which are normally refused, without downloading them: a way to see what scope is excluding.</td></tr>
<tr><td><tt>--sitemap (-%m), --sitemap-url URL (-%mu)</tt></td><td>Also take start URLs from the site's sitemap, for pages nothing links to. Off by default.</td></tr>
</table>
<p>Link-following only finds what something links to. Anything a site publishes
solely in its sitemap is invisible to HTTrack unless you ask for it.
<tt>--sitemap</tt> reads the start host's <tt>robots.txt</tt> for
<tt>Sitemap:</tt> lines and falls back to <tt>/sitemap.xml</tt>;
<tt>--sitemap-url</tt> names one directly. Nested <tt>sitemapindex</tt> files
and gzipped <tt>.xml.gz</tt> sitemaps are followed. The URLs found become start
URLs with the full depth budget, but they still go through your filters and
scope rules, so a sitemap cannot widen a crawl you deliberately narrowed. It is
off by default because a sitemap can list thousands of pages nothing links
to.</p>
<p>One surprise worth knowing: a sitemap you name with <tt>--sitemap-url</tt>,
and one the site itself declares in <tt>robots.txt</tt>, are fetched even when
<tt>robots.txt</tt> disallows that path, because naming or declaring a sitemap
is an invitation to read it. Only the guessed <tt>/sitemap.xml</tt> obeys a
<tt>Disallow</tt>. The URLs listed inside are gated normally either way.</p>
<p>The single most common surprise is "only the home page came down." That is
usually not a scope option at all: it is an off-host redirect. A start URL of
<tt>http://example.com/</tt> that redirects to <tt>https://www.example.com/</tt>
lands you on a different host, and same-host scope stops the crawl there. Start
from the final URL, or add a filter that re-admits the real host (see
<a href="#filters">Filters</a>). The log will show the redirect.</p>
<p><tt>-n</tt> is the fix for pages that render locally without their images or
stylesheets: it lets HTTrack pull in requisites that sit just outside scope. Note
that its embedded-asset handling (following <tt>img</tt>, <tt>link</tt>,
<tt>script</tt>, <tt>style</tt> and HTML5 <tt>source</tt>/<tt>track</tt> targets
past the normal depth and filter limits) applies only when <tt>-n</tt> is on; it
is not automatic. It can also over-fetch by dragging in a whole external host from
a single link, in which case name the assets you want with a filter instead.</p>
<h3 id="filters">3. Filters and scan rules</h3>
<p>Filters are the number-one source of confusion, and also the tool that solves
most scope problems once you understand them. A filter is a rule that accepts
(<tt>+</tt>) or rejects (<tt>-</tt>) URLs by pattern. The sign is mandatory:
<tt>+pattern</tt> adds, <tt>-pattern</tt> removes, and a bare pattern is an error.</p>
<p>The rules that matter:</p>
<ul>
<li><b>Last match wins.</b> Rules are applied in order and the last one that
matches a URL decides its fate. Order your rules from general to specific.</li>
<li><b>Wildcards.</b> <tt>*</tt> matches any run of characters;
<tt>*[a-z]</tt>, <tt>*[0-9]</tt> and similar classes match sets. So
<tt>+*.pdf</tt> means "any URL ending in .pdf".</li>
<li><b>Whitelisting.</b> To keep one site and nothing else, deny everything then
re-admit the host: <tt>"-*" "+example.com/*"</tt>. A lone <tt>+</tt> rule only
adds to the default scope; it never restricts.</li>
<li><b>Size rules.</b> <tt>*[&gt;100000]</tt> and <tt>*[&lt;1000]</tt> filter by
byte size. Because size is only known once the transfer starts, an oversize file
is fetched partway and then aborted, not skipped for free.</li>
<li><b>mime: rules.</b> A rule like <tt>-mime:video/*</tt> matches the
<tt>Content-Type</tt>. That type is only known after the response headers arrive,
so a mime rule <b>cannot stop a request</b>; it can only abort the body. Use a
URL pattern when you want to avoid the fetch entirely.</li>
</ul>
<p>Quote your filters. Shells treat <tt>*</tt>, <tt>[</tt> and sometimes <tt>+</tt>
specially, so wrap each rule in quotes as shown above. The full pattern language,
with tables for wildcards, size and mime, is in
<a href="filters.html">the filters page</a>, and the
<a href="faq.html">FAQ</a> has a worked tutorial.</p>
<p><b>robots.txt.</b> By default HTTrack obeys <tt>robots.txt</tt> (<tt>-s2</tt>).
<tt>-s0</tt> ignores it entirely, <tt>-s1</tt> obeys it but lets one of your
<tt>+</tt> filters override a disallow for a URL you explicitly asked for. Note
that a <tt>403 Forbidden</tt> is a server refusal, not a robots rule: robots
options will not help there. That is an
<a href="#identity">identity</a> problem.</p>
<h4>Filter wildcards</h4>
<p>Inside a filter pattern, <tt>*</tt> matches any run of characters; a few
bracket forms match narrower sets. The full table, with size and mime rules, is on
<a href="filters.html">the filters page</a>.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Wildcard</b></td><td><b>Matches</b></td><td><b>Example</b></td></tr>
<tr><td><tt>*</tt></td><td>any run of characters</td><td><tt>+*.pdf</tt> &mdash; any URL ending <tt>.pdf</tt></td></tr>
<tr><td><tt>*[file]</tt>, <tt>*[name]</tt></td><td>one path segment (any char but <tt>/</tt> and <tt>?</tt>)</td><td><tt>example.com/*[file]/</tt> &mdash; a directory-index page</td></tr>
<tr><td><tt>*[path]</tt></td><td>a path, slashes allowed (any char but <tt>?</tt>)</td><td><tt>example.com/*[path].zip</tt></td></tr>
<tr><td><tt>*[param]</tt></td><td>an optional query string</td><td><tt>page.html*[param]</tt> matches with or without <tt>?...</tt></td></tr>
<tr><td><tt>*[a,b,c]</tt></td><td>any one character in the set</td><td><tt>*[a,b,c].txt</tt></td></tr>
<tr><td><tt>*[a-z]</tt></td><td>any one character in the range</td><td><tt>img*[0-9].gif</tt></td></tr>
<tr><td><tt>*[\x]</tt></td><td>the literal character x (escapes <tt>* [ ] \</tt>)</td><td><tt>*[\*]</tt> matches a real <tt>*</tt></td></tr>
<tr><td><tt>*[&lt;NN]</tt>, <tt>*[&gt;NN]</tt></td><td>file size in KB below / above NN</td><td><tt>-*.gif*[&lt;5]</tt> skips GIFs under 5&nbsp;KB</td></tr>
<tr><td><tt>*[]</tt></td><td>end anchor: nothing may follow</td><td><tt>*.html*[]</tt> rejects <tt>i.html?p=1</tt></td></tr>
</table>
<h3 id="limits">4. Limits and politeness</h3>
<p>HTTrack ships cautious on purpose: it is easy to hammer a small site by
accident, and the <a href="abuse.html">abuse page</a> is worth a read. The limits
below let you go faster when you own the target, and slower when you do not.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--max-rate (-A)</tt></td><td>Maximum transfer rate in bytes/sec. <b>The default is about 100 KB/s even without this flag.</b> Raise it to go faster.</td></tr>
<tr><td><tt>--sockets (-c)</tt></td><td>Number of parallel connections (default 4). <tt>--tiny</tt>, <tt>--wide</tt> and <tt>--ultrawide</tt> are presets.</td></tr>
<tr><td><tt>--connection-per-second (-%c)</tt></td><td>New connections opened per second (default 5).</td></tr>
<tr><td><tt>--max-size (-M)</tt></td><td>Stop after N bytes <b>received from the network</b> across the whole mirror (this counts what was transferred, not what was saved).</td></tr>
<tr><td><tt>--max-time (-E)</tt></td><td>Stop after N seconds of wall-clock time.</td></tr>
<tr><td><tt>--max-files (-m)</tt></td><td>Per-file size caps.</td></tr>
<tr><td><tt>--timeout (-T), --retries (-R), --min-rate (-J), --host-control (-H)</tt></td><td>Idle timeout, retry count, minimum acceptable rate, and host-ban behavior for slow or dead hosts.</td></tr>
<tr><td><tt>--max-pause (-G), --pause (-%G)</tt></td><td>Pause the mirror at N bytes, or pause between files, to spread the load.</td></tr>
</table>
<p><b>The security clamps.</b> To keep an accidental typo from turning into a flood,
HTTrack silently caps a few values: at most 8 connections (<tt>-c</tt>), at most
10 MB/s (<tt>-A</tt>), and at most 5 new connections per second (<tt>-%c</tt>).
Ask for more and you get the ceiling, quietly. The single flag
<tt>--disable-security-limits</tt> lifts all three (the short form <tt>-%!</tt>
also works, but the bare <tt>!</tt> is awkward to type safely in a shell). Use it
only against infrastructure you are allowed to load that hard.</p>
<h3 id="names">5. File names and types</h3>
<p>Where local files land, and what they are called, is controlled by the naming
options. This is the second-biggest source of "why did it do that" questions,
usually about a URL like <tt>/article?id=42</tt> or a <tt>.php</tt> page that is
really HTML.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--structure (-N)</tt></td><td>The local path and name layout. Presets are numeric, and you can also give a template such as <tt>--structure "%h%p/%n%q.%t"</tt>.</td></tr>
<tr><td><tt>--long-names (-L)</tt></td><td>Long names, 8.3 names, or ISO9660 for CD masters.</td></tr>
<tr><td><tt>--assume (-%A)</tt></td><td>Assume a MIME type for an extension, for example <tt>--assume php=text/html</tt>. This also skips the extra HEAD probe HTTrack would otherwise send to learn the type.</td></tr>
<tr><td><tt>--delayed-type-check (-%N), --cached-delayed-type-check (-%D), --check-type (-u), -%t</tt></td><td>When and how the content type is checked, and whether the original extension is kept.</td></tr>
<tr><td><tt>--include-query-string (-%q), --strip-query (-%g)</tt></td><td>Whether the query string appears in the local filename, and whether query keys are stripped when deciding if two URLs are the same file.</td></tr>
</table>
<p>The <tt>-N</tt> presets are built from modular arithmetic on the name fields, so
undocumented number combinations often "work" by accident. If you care about the
exact layout, use an explicit template (the <tt>%h %p %n %q %t</tt> placeholders)
rather than a magic number, and check the result on a small crawl first.</p>
<p>A dynamic page served as <tt>.php</tt> or <tt>.asp</tt> that is actually HTML is
the classic case: without help it can be saved with an extension a browser will
not open locally. <tt>--assume php=text/html</tt> fixes both the extension and the
naming.</p>
<h3 id="links">6. Links and page building</h3>
<p>After a page is downloaded, HTTrack parses it for more links and rewrites the
ones it kept so the local copy browses offline. These options tune both halves.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--keep-links (-K)</tt></td><td>How links are rewritten in saved pages. The numbering is inverted from what you might guess: bare <tt>-K</tt> keeps <b>absolute</b> URLs, and <tt>-K0</tt> is the <b>relative</b> default. <tt>-K3</tt> keeps absolute URIs, <tt>-K4</tt> keeps the original links.</td></tr>
<tr><td><tt>--replace-external (-x), --generate-errors (-o)</tt></td><td>Replace external links with an error page, and generate an error page for links that failed.</td></tr>
<tr><td><tt>--preserve (-%p), --disable-passwords (-%x)</tt></td><td>Leave HTML untouched (no rewriting), and strip passwords out of saved links.</td></tr>
<tr><td><tt>--extended-parsing (-%P), --parse-java (-j)</tt></td><td>Aggressive link discovery, and how much script content is parsed for links.</td></tr>
<tr><td><tt>--mime-html (-%M)</tt></td><td>Save the whole mirror as a single MIME-encapsulated <tt>.mht</tt> archive (<tt>index.mht</tt>).</td></tr>
<tr><td><tt>--single-file (-%Z), --single-file-max-size N</tt></td><td>Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded as <tt>data:</tt> URIs. Assets over the cap (10&nbsp;MB by default) keep their link, as do audio, video, and the links from one page to another. A sibling of <tt>-%M</tt>, not a replacement: see the recipe below for which to pick.</td></tr>
<tr><td><tt>--index (-I), --build-top-index (-%i), --search-index (-%I)</tt></td><td>Build a per-mirror index, a top index across projects, and a searchable keyword index.</td></tr>
</table>
<p>HTTrack finds links by parsing HTML and CSS. It does not run JavaScript, so any
URL a page builds at runtime in script (a lazy-loaded image, a
JavaScript-assembled path) is invisible to the crawler and will be missing from
the mirror. There is no flag that fixes this; the asset has to appear in the
static HTML or CSS to be found. <tt>-%P</tt> widens discovery for links that are
present but awkwardly formatted, not for links that do not exist until script
runs.</p>
<h3 id="identity">7. Identity, cookies and login</h3>
<p>By default HTTrack is an honest robot: it sends a <tt>User-Agent</tt> of
<tt>HTTrack</tt>, a Referer with each link (which reveals the crawl path to the
server), and obeys robots. Plenty of sites filter exactly that profile. These
options control what HTTrack says about itself.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--user-agent (-F)</tt></td><td>The <tt>User-Agent</tt>. Set a browser string to get past crawler blocks; <tt>--user-agent ""</tt> sends none.</td></tr>
<tr><td><tt>--referer (-%R), --from (-%E), --language (-%l), --accept (-%a)</tt></td><td>Referer, From, Accept-Language and Accept headers.</td></tr>
<tr><td><tt>--headers (-%X)</tt></td><td>Add raw header lines to every request.</td></tr>
<tr><td><tt>--footer (-%F)</tt></td><td>A footer written into saved pages (on disk, not a network header). See <b>Footer fields</b> below.</td></tr>
<tr><td><tt>--cookies (-b), --cookies-file (-%K)</tt></td><td>Accept cookies, and preload a Netscape <tt>cookies.txt</tt>.</td></tr>
</table>
<p><b>Footer fields.</b> A footer with no <tt>%s</tt> may reference named fields:
<tt>{addr}</tt>, <tt>{path}</tt>, <tt>{url}</tt>, <tt>{date}</tt> (mirror time),
<tt>{lastmodified}</tt> (the page's Last-Modified), <tt>{version}</tt>,
<tt>{mime}</tt>, <tt>{charset}</tt>, <tt>{status}</tt> and <tt>{size}</tt>; write
<tt>{{</tt> or <tt>}}</tt> for a literal brace. A footer that contains <tt>%s</tt>
keeps the older positional form (host, path, date in that order). Example:
<tt>-%F "&lt;!-- Mirrored from {url} on {date} --&gt;"</tt>.</p>
<p><b>Login.</b> For HTTP Basic auth, put the credentials in the URL:
<tt>http://user:pass@host/</tt>. An <tt>@</tt> inside the username must be written
<tt>%40</tt>. Only Basic is supported, not Digest.</p>
<p>For cookie or form logins, the simplest path is to log in with a browser, export
its <tt>cookies.txt</tt>, and drop that file in the project directory so HTTrack
sends the session cookie. For a form that needs a POST, <tt>--catchurl</tt> can
capture the exact request your browser sends and replay it. A few cookie caveats
to know: expiry is ignored, there is a silent cap of about 8 cookies sent per
request, and <tt>-b0</tt> disables cookies and the reuse of Basic credentials
across links at the same time.</p>
<h3 id="proxy">8. Proxy and network</h3>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--proxy (-P)</tt></td><td>Route through a proxy. HTTP, SOCKS5 and CONNECT are supported: <tt>-P host:8080</tt>, <tt>-P socks5://host:1080</tt>, <tt>-P connect://host:443</tt>, with optional <tt>user:pass@</tt>.</td></tr>
<tr><td><tt>--httpproxy-ftp (-%f)</tt></td><td>Send FTP requests through the HTTP proxy.</td></tr>
<tr><td><tt>--protocol (-@i)</tt></td><td>Prefer IPv4 or IPv6.</td></tr>
<tr><td><tt>--http-10 (-%h), --keep-alive (-%k), --disable-compression (-%z)</tt></td><td>Force HTTP/1.0 (drops keep-alive and compression, useful for fragile CGI), toggle keep-alive, and toggle compression.</td></tr>
<tr><td><tt>--bind (-%b), --tolerant (-%B)</tt></td><td>Bind to a local address, and accept technically-bogus responses some servers send.</td></tr>
</table>
<p>Two network facts worth stating plainly. Over SOCKS5, HTTrack always resolves
host names at the proxy (remote DNS) for both <tt>socks5://</tt> and
<tt>socks5h://</tt>, so your local resolver is never consulted. And HTTrack does
not verify TLS certificates: HTTPS gives you an encrypted transport, but not an
authenticated one. That is a deliberate choice for a mirroring tool, not a bug,
but it is worth knowing if you are relying on it for trust.</p>
<h3 id="update">9. Update and cache</h3>
<p>Every project keeps a cache under <tt>hts-cache/</tt>. It records every URL that
was fetched, together with the options you used, and it is what makes resuming and
updating possible. It is not a size-limited scratch area you can delete: throw it
away and you lose the ability to continue or update the mirror.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--continue</tt></td><td>Carry on an interrupted mirror, trusting the cache: it does not re-check pages already stored.</td></tr>
<tr><td><tt>--update</tt></td><td>Re-run the mirror, revalidating each page with the server (If-Modified-Since / If-None-Match) and downloading only what changed.</td></tr>
<tr><td><tt>--purge-old=0 (-X0)</tt></td><td>Do not purge. By default an update deletes local files that are no longer part of the mirror; <tt>--purge-old=0</tt> keeps them.</td></tr>
<tr><td><tt>--cache (-C)</tt></td><td>Cache mode. The default already does the right thing and switches to update-checking when it detects an existing mirror.</td></tr>
<tr><td><tt>--urlhack (-%u), --keep-www-prefix (-%j), --keep-double-slashes (-%o), --keep-query-order (-%y), --do-not-recatch (-%n), --updatehack (-%s), --store-all-in-cache (-k)</tt></td><td>URL-deduplication behavior and cache storage details.</td></tr>
<tr><td><tt>--debug-cache (-#C), --repair-cache (-#R), --clean</tt></td><td>Inspect the cache, repair its ZIP, and erase cache plus logs.</td></tr>
</table>
<p><b>The purge trap.</b> An <tt>--update</tt> run rebuilds the list of files the
mirror should contain, then deletes any previously-mirrored file that is not on the
new list. This is what keeps a mirror in sync with a shrinking site, but it means a
partial or interrupted update can delete files you meant to keep. If an update might
not complete cleanly, add <tt>-X0</tt> to protect the existing tree, and expect
dynamic pages to look "changed" on every run. See the
<a href="cache.html">cache page</a> for the details.</p>
<h3 id="experts">10. Experts and scripting</h3>
<p>HTTrack is also a scriptable fetch-and-scan tool. These options turn off the
mirror behavior and expose the engine.</p>
<table class="tblRegular tableWidth" border="0">
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
<tr><td><tt>--get URL</tt></td><td>Fetch a single file and stop. Cache, index, depth, cookies and robots are all off for this mode.</td></tr>
<tr><td><tt>--spider --testlinks --skeleton</tt></td><td>Scan without saving, test links at depth 1, or keep HTML only. Handy for checking a site before a real crawl.</td></tr>
<tr><td><tt>--userdef-cmd (-V)</tt></td><td>Run a shell command on each downloaded file; <tt>$0</tt> is the file path. Good for on-the-fly processing.</td></tr>
<tr><td><tt>--callback (-%W)</tt></td><td>Load an external callback module to hook the engine.</td></tr>
<tr><td><tt>--do-not-log (-Q), --quiet (-q), --verbose (-v), --file-log (-f), --extra-log (-z), --debug-log (-Z)</tt></td><td>Logging: quiet, no questions, verbose on screen, and the various log-to-file levels.</td></tr>
</table>
<p>The <a href="dev.html">developer page</a> covers the callback API and batch use
in more depth.</p>
<h3 id="recipes">11. Recipes</h3>
<p>Copy-ready command lines for the tasks people ask about most. Each has the one
gotcha that trips it up.</p>
<h4>Mirror one site, nothing off-host</h4>
<p><tt>httrack https://example.com/ --path mydir</tt><br>
<small>Same-host is already the default. The usual failure is a
<tt>www.</tt>-to-apex or <tt>http</tt>-to-<tt>https</tt> redirect that moves you off
host and stops the crawl; start from the final URL, or add
<tt>"+finalhost/*"</tt>.</small></p>
<h4>One level of links only</h4>
<p><tt>httrack https://example.com/ --depth=2 --path mydir</tt><br>
<small>Depth counts the start page as level 1, so one level out is <tt>--depth=2</tt>.</small></p>
<h4>This site only, deny everything else</h4>
<p><tt>httrack https://example.com/ "-*" "+example.com/*" --path mydir</tt><br>
<small>You must deny-all first; a lone <tt>+</tt> only adds, and the last matching
rule wins.</small></p>
<h4>Download the PDFs on a site</h4>
<p><tt>httrack https://example.com/ "-*" "+https://example.com/*.html" "+https://example.com/*[path]/" "+https://example.com/*.pdf" --path mydir</tt><br>
<small>HTTrack finds PDFs by parsing the site's HTML, so a plain
<tt>"-*" "+example.com/*.pdf"</tt> is wrong: it prunes the pages that carry the
links and keeps only PDFs reachable from the front page. Instead admit the HTML as
scaffolding (<tt>*.html</tt> and <tt>*[path]/</tt> for directory-index pages at any
depth, e.g. <tt>docs/</tt> or <tt>a/b/deep/</tt>; <tt>*[file]/</tt> would stop at one
level), keep the PDFs, and let <tt>-*</tt> drop everything else (images,
archives, off-site assets). PDFs on another host (a CDN or docs subdomain) are not
included by default; allow that host too, e.g. <tt>"+docs.example.com/*.pdf"</tt>,
or widen to <tt>"+*.pdf"</tt> for PDFs anywhere.</small></p>
<h4>Keep page requisites, including off-host images</h4>
<p><tt>httrack https://example.com/blog/ --near --path mydir</tt><br>
<small><tt>--near</tt> can over-fetch by pulling in an entire external host from one
link; if it does, name the assets with <tt>"+cdn.example.com/*"</tt> instead.</small></p>
<h4>Resume an interrupted crawl</h4>
<p><tt>httrack --continue --path mydir</tt><br>
<small>Needs an intact <tt>hts-cache/</tt>. Deleting it loses the URL list and your
options.</small></p>
<h4>Update a mirror, downloading only what changed</h4>
<p><tt>httrack --update --path mydir</tt><br>
<small>Purge deletes any previously-mirrored file not seen this run; add
<tt>--purge-old=0</tt> to protect the tree against a partial update.</small></p>
<h4>Reach a section behind a login cookie</h4>
<p><small>Export your browser's <tt>cookies.txt</tt> into the project directory,
then:</small><br>
<tt>httrack https://example.com/members/ --path mydir</tt><br>
<small>The file must be Netscape <tt>cookies.txt</tt> in the project folder;
<tt>--cookies=0</tt> would disable cookies and Basic-auth reuse together.</small></p>
<h4>Get past a crawler-UA block</h4>
<p><tt>httrack https://example.com/ --user-agent "Mozilla/5.0 (Windows NT 10.0)" --path mydir</tt><br>
<small>A 403 is server-side, not robots: change the User-Agent, and add
<tt>--http-10</tt> for fragile CGI. Do not reach for <tt>--robots=0</tt>.</small></p>
<h4>Full-speed mirror on your own server</h4>
<p><tt>httrack https://example.com/ --max-rate=2000000 --sockets=8 --disable-security-limits --path mydir</tt><br>
<small>The default throttles to about 100 KB/s. <tt>--disable-security-limits</tt>
lifts the built-in caps; use it only against infrastructure you are allowed to
load.</small></p>
<h4>Save a WARC archive of the crawl</h4>
<p><tt>httrack https://example.com/ --warc --path mydir</tt><br>
<small>Writes a standard WARC/1.1 file (<tt>httrack-&lt;timestamp&gt;.warc.gz</tt>) in
the project folder alongside the browsable mirror, not instead of it. Set the name
with <tt>--warc-file NAME</tt> and split a large crawl with <tt>--warc-max-size N</tt>;
add <tt>--warc-cdx</tt> for a sorted CDXJ index, or <tt>--wacz</tt> to bundle the
archive, index and pages into one WACZ for replay tools such as
replayweb.page.</small></p>
<h4>See what a re-crawl changed</h4>
<p><tt>httrack https://example.com/ --update --changes --path mydir</tt><br>
<small>Writes <tt>hts-changes.json</tt> in the project folder, listing every
mirrored file as new, changed, unchanged or gone, plus a one-line summary in the
log. &quot;Changed&quot; means the bytes really differ: a server that answers 200
with the same content it served last time lands in <tt>unchanged</tt>. Deletions
are reported whether or not <tt>--purge-old</tt> is deleting them. The format is
documented in <a href="changes.html">the change report specification</a>.</small></p>
<h4>Pages you can hand to someone as one file</h4>
<p><tt>httrack https://example.com/ --single-file --path mydir</tt><br>
<small>Rewrites each saved page after the crawl so its stylesheets, scripts,
images and fonts are embedded as <tt>data:</tt> URIs. The mirror stays a normal
browsable tree, with links between pages relative and the assets still on disk,
but any single <tt>.html</tt> file can now be mailed or archived on its own.
Raise or lower the 10&nbsp;MB per-asset limit with
<tt>--single-file-max-size N</tt>; anything over it, plus audio and video, keeps
its link. One caveat if you raise it: an inlined stylesheet becomes a
<tt>data:</tt> URL, whose path is opaque, so an asset it referenced that stayed
over the cap no longer resolves from inside it. Raising the cap past that asset
embeds it too and the question goes away.<br>
Reach for this when the file has to open for someone you cannot make assumptions
about: it is plain HTML and needs no add-on. Reach for <tt>--mime-html</tt>
instead when a Chromium-family browser is a given and the mirror is large: MIME
carries text parts without the base64 tax, keeps each resource's original URL,
and stores a shared asset once rather than re-embedding it in every page that
uses it.</small></p>
<h4>HTTrack as a fetch tool</h4>
<p><tt>httrack --get https://host/file.bin --path tmp</tt><br>
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
all disabled.</small></p>
<p><br>For the complete, always-current option list, see
<a href="httrack.man.html">the manual page</a>. For the filter language, see
<a href="filters.html">filters</a>; for the cache and updates, see
<a href="cache.html">cache</a>; for the change report, see
<a href="changes.html">changes</a>.</p>
<!-- doc-chrome:bottom -->
</main>
</div>
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
<footer>&copy; 1998-2026 Xavier Roche &amp; other contributors - Web Design: Leto Kauler.</footer>
<!-- /doc-chrome:bottom -->
</body>
</html>

408
html/contact.html
View File

@@ -1,75 +1,120 @@
<!DOCTYPE html>
<html lang="en">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<!-- doc-chrome:head -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="How to reach the HTTrack project, and who contributed to it.">
<link rel="stylesheet" href="doc.css">
<script src="doc.js" defer></script>
<!-- /doc-chrome:head -->
<title>Contact and credits</title>
<script language="javascript">
<!--
function FOS(host,nom,info) {
var s;
if (info == "") info=nom+"@"+host;
s="mail";
document.write("<a href='"+s+"to:"+nom+"@"+host+"'>"+info+"</a>");
}
// -->
</script>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
<title>HTTrack Website Copier - Offline Browser</title>
<script language="javascript">
<!--
function FOS(host,nom,info) {
var s;
if (info == "") info=nom+"@"+host;
s="mail";
document.write("<a href='"+s+"to:"+nom+"@"+host+"'>"+info+"</a>");
}
// -->
</script>
<style type="text/css">
<!--
body {
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
background: #77b;
}
body, td {
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
}
#subTitle {
background: #000; color: #fff; padding: 4px; font-weight: bold;
}
#siteNavigation a, #siteNavigation .current {
font-weight: bold; color: #448;
}
#siteNavigation a:link { text-decoration: none; }
#siteNavigation a:visited { text-decoration: none; }
#siteNavigation .current { background-color: #ccd; }
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
a:link { text-decoration: underline; color: #00f; }
a:visited { text-decoration: underline; color: #000; }
a:hover { text-decoration: underline; color: #c00; }
a:active { text-decoration: underline; }
#pageContent {
clear: both;
border-bottom: 6px solid #000;
padding: 10px; padding-top: 20px;
line-height: 1.65em;
background-image: url(images/bg_rings.gif);
background-repeat: no-repeat;
background-position: top right;
}
#pageContent, #siteNavigation {
background-color: #ccd;
}
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
h1 { margin: 0; font-weight: bold; font-size: 2em; }
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
.blak { background-color: #000; }
.hide { display: none; }
.tableWidth { min-width: 400px; }
.tblRegular { border-collapse: collapse; }
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
.tblNoBorder td { border: 0; }
// -->
</style>
</head>
<body>
<!-- doc-chrome:top -->
<a class="skip" href="#main">Skip to content</a>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr>
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
<tr>
<td id="subTitle">Open Source offline browser</td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr class="blak">
<td>
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
<tr>
<td colspan="6">
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
<tr>
<td id="pageContent">
<!-- ==================== End prologue ==================== -->
<header class="masthead">
<img src="images/wordmark.svg" width="400" height="36" alt="HTTrack Website Copier">
<div class="tagline">Open Source offline browser</div>
</header>
<div class="wrap">
<nav class="toc" aria-label="Documentation">
<h2>Start here</h2>
<ul>
<li><a href="guide.html">Interface guide</a></li>
<li><a href="faq.html">FAQ and troubleshooting</a></li>
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
</ul>
<h2>Command line</h2>
<ul>
<li><a href="cmdguide.html">Command-line guide</a></li>
<li><a href="httrack.man.html">Option reference</a></li>
<li><a href="filters.html">Filter syntax</a></li>
</ul>
<h2>Developers</h2>
<ul>
<li><a href="dev.html">Programming</a></li>
<li><a href="library.html">libhttrack API</a></li>
<li><a href="plug.html">Callbacks</a></li>
<li><a href="scripting.html">Scripting</a></li>
<li><a href="cache.html">Cache format</a></li>
<li><a href="changes.html">Change report format</a></li>
</ul>
<h2>More</h2>
<ul>
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
<li><a href="contact.html" aria-current="page" class="here">Contact and credits</a></li>
<li><a href="index.html">All documentation</a></li>
</ul>
</nav>
<main id="main">
<!-- /doc-chrome:top -->
<h1>Contact Us!</h1>
<h2 align="center"><em>Contact Us!</em></h2>
<br>
<h3 id="please-visit-our-website">
<h3>
<center>
<a href="http://www.httrack.com/" target="_blank">Please visit our website!</a>
<br>
@@ -82,94 +127,94 @@
<pre>
Informations on this project:
This project has been developed by:
Xavier Roche (xroche at httrack dot com)
for the main engine and Windows interface
and maintainer for v2.0 and v3.0
Yann Philippot (yphilippot at lemel dot fr)
past contributor (java binary .class parser)
David Lawrie (dalawrie at lineone dot net)
Robert Lagadec (rlagadec at yahoo dot fr)
for checking both English & French translations
Juan Pablo Barrio Lera (University of León)
for Spanish translations
Rainer Klueting (rainer at klueting dot de)
Bastian Gorke (bastiang at yahoo dot com)
Rudi Ferrari (Wyando at netcologne dot de)
Marcus Gaza (MarcusGaza at t-online dot de)
for German translations
Rudi Ferrari (Wyando at netcologne dot de)
for Dutch translations
Lukasz Jokiel (Opole University of Technology, Lukasz dot Jokiel at po dot opole dot pl)
for Polish translations
Rui Fernandes (CANTIC, ruiefe at mail dot malhatlantica dot pt)
Pedro T dot Pinheiro (Universidade Nova de Lisboa-FCT, ptiago at mail dot iupi dot pt)
for Portuguese translations
Andrei Iliev (iliev at vitaplus dot ru)
for Russian translations
Witold Krakowski (wtkrak at netscape dot net )
for Italian translations
Jozsef Tamas Herczeg (hdodi at freemail dot hu)
for Hungarian translation
Paulo Neto (company at layout dot com dot br)
for Brazilian translation
Brook Qin (brookqwr at sina dot com)
for simplified Chinese translation
David Hing Cheong Hung (DAVEHUNG at mtr dot com dot hk)
Addy Lin (addy1975 at pchome dot com dot tw)
for traditional Chinese translation
Jesper Bramm (bramm at get2net dot dk)
for Danish translation
Tõnu Virma
for Estonian translation
Staffan Ström (staffan at fam-strom dot org)
for Swedish translation
Mehmet Akif Köeoðlu (mak at ttnet dot net dot tr)
for Turkish translation
Aleksandar Savic (aleks at macedonia dot eu dot org)
for Macedonian translation
Takayoshi Nakasikiryo
for Japanese translation
Martin Sereday (sereday at slovanet dot sk)
for Slovak translation
Antonín Matìjèík (matejcik at volny dot cz)
for Czech translation
Andrij Shevchuk (http://programy dot com dot ua)
for Ukrainian translation
Tobias "Spug" Langhoff (spug_enigma at hotmail dot com)
for Norwegian translation
Jadran Rudeciur (jrudec at email dot si)
for Slovenian translation
Alin Gheorghe Miron (miron dot alin at personal dot ro)
for Romanian translation
Michael Papadakis (mikepap at freemail dot gr)
for Greek translation
Thanks to:
Leto Kauler (molotov at tasmail dot com)
for the site/logos design
Special Thanks to:
Patrick Ducrot & Daniel Carré (ENSI of Caen)
for their initial support
Fred Cohen (fc at all dot net)
for HTTrack user's guide
Greetings to:
Christian Marillat (marillat dot christian at wanadoo dot fr)
for autoconf compliance and .deb package
Jean-loup Gailly from gzip dot org
for the help provided with Zlib
Eric A dot Young (eay at cryptsoft dot com)
for SSL cryptographic material (OpenSSL)
Russ Freeman from gipsymedia
for hints on DLL dynamic load
Paul DiLascia
for helping to fix problems with CHtmlView
ISMRA/Ensi of Caen
for their initial support
.. and all users that are using and supporting HTTrack!
This project has been developed by:
Xavier Roche (xroche at httrack dot com)
for the main engine and Windows interface
and maintainer for v2.0 and v3.0
Yann Philippot (yphilippot at lemel dot fr)
for the java binary dot class parser
David Lawrie (dalawrie at lineone dot net)
Robert Lagadec (rlagadec at yahoo dot fr)
for checking both English & French translations
Juan Pablo Barrio Lera (University of León)
for Spanish translations
Rainer Klueting (rainer at klueting dot de)
Bastian Gorke (bastiang at yahoo dot com)
Rudi Ferrari (Wyando at netcologne dot de)
Marcus Gaza (MarcusGaza at t-online dot de)
for German translations
Rudi Ferrari (Wyando at netcologne dot de)
for Dutch translations
Lukasz Jokiel (Opole University of Technology, Lukasz dot Jokiel at po dot opole dot pl)
for Polish translations
Rui Fernandes (CANTIC, ruiefe at mail dot malhatlantica dot pt)
Pedro T dot Pinheiro (Universidade Nova de Lisboa-FCT, ptiago at mail dot iupi dot pt)
for Portuguese translations
Andrei Iliev (iliev at vitaplus dot ru)
for Russian translations
Witold Krakowski (wtkrak at netscape dot net )
for Italian translations
Jozsef Tamas Herczeg (hdodi at freemail dot hu)
for Hungarian translation
Paulo Neto (company at layout dot com dot br)
for Brazilian translation
Brook Qin (brookqwr at sina dot com)
for simplified Chinese translation
David Hing Cheong Hung (DAVEHUNG at mtr dot com dot hk)
Addy Lin (addy1975 at pchome dot com dot tw)
for traditional Chinese translation
Jesper Bramm (bramm at get2net dot dk)
for Danish translation
Tõnu Virma
for Estonian translation
Staffan Ström (staffan at fam-strom dot org)
for Swedish translation
Mehmet Akif Köeoðlu (mak at ttnet dot net dot tr)
for Turkish translation
Aleksandar Savic (aleks at macedonia dot eu dot org)
for Macedonian translation
Takayoshi Nakasikiryo
for Japanese translation
Martin Sereday (sereday at slovanet dot sk)
for Slovak translation
Antonín Matìjèík (matejcik at volny dot cz)
for Czech translation
Andrij Shevchuk (http://programy dot com dot ua)
for Ukrainian translation
Tobias "Spug" Langhoff (spug_enigma at hotmail dot com)
for Norwegian translation
Jadran Rudeciur (jrudec at email dot si)
for Slovenian translation
Alin Gheorghe Miron (miron dot alin at personal dot ro)
for Romanian translation
Michael Papadakis (mikepap at freemail dot gr)
for Greek translation
Thanks to:
Leto Kauler (molotov at tasmail dot com)
for the site/logos design
Special Thanks to:
Patrick Ducrot & Daniel Carré (ENSI of Caen)
for their initial support
Fred Cohen (fc at all dot net)
for HTTrack user's guide
Greetings to:
Christian Marillat (marillat dot christian at wanadoo dot fr)
for autoconf compliance and .deb package
Jean-loup Gailly from gzip dot org
for the help provided with Zlib
Eric A dot Young (eay at cryptsoft dot com)
for SSL cryptographic material (OpenSSL)
Russ Freeman from gipsymedia
for hints on DLL dynamic load
Paul DiLascia
for helping to fix problems with CHtmlView
ISMRA/Ensi of Caen
for their initial support
.. and all users that are using and supporting HTTrack!
</pre>
<hr>
@@ -180,35 +225,46 @@ ISMRA/Ensi of Caen
You can also contact by email, but due to the large volume of messages, it is impossible
to always respond (especially for configuration help or other configuration-related
questions).
<br>
<script language="javascript">
<!--
FOS('httrack.com','httrack','httrack at httrack dot com (HTTrack project)');
document.write('<br>');
FOS('httrack.com','roche','roche at httrack dot com (Xavier ROCHE)');
// -->
</script>
<noscript>
httrack at httrack dot com (HTTrack project)<br>
roche at httrack dot com (Xavier ROCHE)<br>
</noscript>
<br>
<script language="javascript">
<!--
FOS('httrack.com','httrack','httrack at httrack dot com (HTTrack project)');
document.write('<br>');
FOS('httrack.com','roche','roche at httrack dot com (Xavier ROCHE)');
// -->
</script>
<noscript>
httrack at httrack dot com (HTTrack project)<br>
roche at httrack dot com (Xavier ROCHE)<br>
</noscript>
<br><hr><br>
<br>
This program is covered by the GNU General Public License.<br>
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2026 Xavier Roche and other contributors
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2007 Xavier Roche and other contributors
<br>
<!-- doc-chrome:bottom -->
</main>
</div>
<!-- ==================== Start epilogue ==================== -->
</td>
</tr>
</table>
</td>
</tr>
</table>
</td>
</tr>
</table>
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
<footer>&copy; 1998-2026 Xavier Roche &amp; other contributors - Web Design: Leto Kauler.</footer>
<!-- /doc-chrome:bottom -->
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
<tr>
<td id="footer"><small>&copy; 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
</tr>
</table>
</body>
</html>

231
html/dev.html
View File

@@ -1,111 +1,158 @@
<!DOCTYPE html>
<html lang="en">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
<head>
<!-- doc-chrome:head -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Using HTTrack from a script, from the callbacks, or through the libhttrack library.">
<link rel="stylesheet" href="doc.css">
<script src="doc.js" defer></script>
<!-- /doc-chrome:head -->
<title>Programming with HTTrack</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
<title>HTTrack Website Copier - Offline Browser</title>
<style type="text/css">
<!--
body {
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
background: #77b;
}
body, td {
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
}
#subTitle {
background: #000; color: #fff; padding: 4px; font-weight: bold;
}
#siteNavigation a, #siteNavigation .current {
font-weight: bold; color: #448;
}
#siteNavigation a:link { text-decoration: none; }
#siteNavigation a:visited { text-decoration: none; }
#siteNavigation .current { background-color: #ccd; }
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
a:link { text-decoration: underline; color: #00f; }
a:visited { text-decoration: underline; color: #000; }
a:hover { text-decoration: underline; color: #c00; }
a:active { text-decoration: underline; }
#pageContent {
clear: both;
border-bottom: 6px solid #000;
padding: 10px; padding-top: 20px;
line-height: 1.65em;
background-image: url(images/bg_rings.gif);
background-repeat: no-repeat;
background-position: top right;
}
#pageContent, #siteNavigation {
background-color: #ccd;
}
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
h1 { margin: 0; font-weight: bold; font-size: 2em; }
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
.blak { background-color: #000; }
.hide { display: none; }
.tableWidth { min-width: 400px; }
.tblRegular { border-collapse: collapse; }
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
.tblNoBorder td { border: 0; }
// -->
</style>
</head>
<body>
<!-- doc-chrome:top -->
<a class="skip" href="#main">Skip to content</a>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr>
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
<tr>
<td id="subTitle">Open Source offline browser</td>
</tr>
</table>
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
<tr class="blak">
<td>
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
<tr>
<td colspan="6">
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
<tr>
<td id="pageContent">
<!-- ==================== End prologue ==================== -->
<header class="masthead">
<img src="images/wordmark.svg" width="400" height="36" alt="HTTrack Website Copier">
<div class="tagline">Open Source offline browser</div>
</header>
<div class="wrap">
<nav class="toc" aria-label="Documentation">
<h2>On this page</h2>
<ul>
<li><a href="#programming">Programming</a></li>
<li><a href="#technical-references">Technical references</a></li>
</ul>
<h2>Start here</h2>
<ul>
<li><a href="guide.html">Interface guide</a></li>
<li><a href="faq.html">FAQ and troubleshooting</a></li>
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
</ul>
<h2>Command line</h2>
<ul>
<li><a href="cmdguide.html">Command-line guide</a></li>
<li><a href="httrack.man.html">Option reference</a></li>
<li><a href="filters.html">Filter syntax</a></li>
</ul>
<h2>Developers</h2>
<ul>
<li><a href="dev.html" aria-current="page" class="here">Programming</a></li>
<li><a href="library.html">libhttrack API</a></li>
<li><a href="plug.html">Callbacks</a></li>
<li><a href="scripting.html">Scripting</a></li>
<li><a href="cache.html">Cache format</a></li>
<li><a href="changes.html">Change report format</a></li>
</ul>
<h2>More</h2>
<ul>
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
<li><a href="contact.html">Contact and credits</a></li>
<li><a href="index.html">All documentation</a></li>
</ul>
</nav>
<main id="main">
<!-- /doc-chrome:top -->
<h1>HTTrack Programming page</h1>
<h2 align="center"><em>HTTrack Programming page</em></h2>
<br>
HTTrack can be used as a third-party program in batch files, or as library.
HTTrack can be used as a third-party program in batch files, or as library.
Depending on your needs, you may look:
<br><br>
<h3>Programming</h3>
<h3 id="programming">Programming</h3>
<ul>
<ul>
<li><a href="scripting.html">Simple use: scripting examples using the httrack commandline program</a></li><br>
You can use tens of options (see <tt>httrack --help</tt>) to control precisely the mirror, and used embedded commands in httrack.
<br><br>
<li><a href="plug.html">More complex use: plugging external C functions to the httrack library</a></li><br>
For advanced functions, you may have to use external C wrappers ; for example when adding advanced crawl features, such as "tuned" filestructure type
<br><i>Important note: please read the license information of httrack before developing add-ons</i><br />
See also the page <a href="plug_330.html">for versions prior to 3.41</a>
<br><br>
<li><a href="library.html">Advanced use: using the library</a></li><br>
The library can be used to write graphical GUIs for httrack, or to run mirrors from a program.
<br><i>Important note: please read the license information of httrack before developing add-ons</i>
</ul>
<h3 id="technical-references">Technical references</h3>
<ul>
<li><a href="cache.html">Cache format</a></li><br>
HTTrack stores original HTML data and references to downloaded files in a cache, located in the hts-cache directory.
This page describes the HTTrack cache format.
<li><a href="changes.html">Change report format</a></li><br>
With --changes, HTTrack writes hts-changes.json describing what the crawl left new, changed, unchanged and gone
compared to the previous mirror. This page describes that file.
</ul>
You can use tens of options (see <tt>httrack --help</tt>) to control precisely the mirror, and used embedded commands in httrack.
<br><br>
<li><a href="plug.html">More complex use: plugging external C functions to the httrack library</a></li><br>
For advanced functions, you may have to use external C wrappers ; for example when adding advanced crawl features, such as "tuned" filestructure type
<br><i>Important note: please read the license information of httrack before developing add-ons</i><br />
See also the page <a href="plug_330.html">for versions prior to 3.41</a>
<br><br>
<li><a href="library.html">Advanced use: using the library</a></li><br>
The library can be used to write graphical GUIs for httrack, or to run mirrors from a program.
<br><i>Important note: please read the license information of httrack before developing add-ons</i>
</ul>
<h3>Technical references</h3>
<ul>
<li><a href="cache.html">Cache format</a></li><br>
HTTrack stores original HTML data and references to downloaded files in a cache, located in the hts-cache directory.
This page describes the HTTrack cache format.
</ul>
<br><br>
<!-- doc-chrome:bottom -->
</main>
</div>
<!-- ==================== Start epilogue ==================== -->
</td>
</tr>
</table>
</td>
</tr>
</table>
</td>
</tr>
</table>
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
<footer>&copy; 1998-2026 Xavier Roche &amp; other contributors - Web Design: Leto Kauler.</footer>
<!-- /doc-chrome:bottom -->
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
<tr>
<td id="footer"><small>&copy; 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
</tr>
</table>
</body>
</html>

349
html/doc.css
View File

@@ -1,349 +0,0 @@
/* Shared chrome for the HTTrack documentation pages. Page-specific rules live
beside the page (guide.css). Read from disk as often as over http, so
everything here has to work from a file:// URL. */
:root {
--field: #77b;
--panel: #ccd;
--accent: #99c;
--ink: #111;
--ink-soft: #444;
--rule: #aab;
--link: #00c;
--link-hover: #c00;
--code-bg: #e8e8f2;
--chip: #dde;
--shadow: rgba(0, 0, 0, .25);
}
@media (prefers-color-scheme: dark) {
:root {
--field: #171a2b;
--panel: #22263a;
--accent: #3b4066;
--ink: #e6e6ee;
--ink-soft: #a9adc4;
--rule: #3b4066;
--link: #9db4ff;
--link-hover: #ff9d9d;
--code-bg: #14172a;
--chip: #2e3350;
--shadow: rgba(0, 0, 0, .6);
}
}
* { box-sizing: border-box; }
/* The plain declaration before each var() is what IE11 gets: it drops the
custom-property line it cannot parse and keeps the literal above it. */
body {
margin: 0;
padding: 0 1rem 3rem;
background: #77b;
background: var(--field);
color: #111;
color: var(--ink);
font: 1em/1.6 "Trebuchet MS", Verdana, system-ui, -apple-system, "Segoe UI", sans-serif;
}
/* ---- page frame ---- */
.masthead, .wrap {
max-width: 68rem;
margin: 0 auto;
}
.masthead {
padding: .6rem 0 0;
}
/* Take back the whitespace the SVG carries around the letters, restoring the old bitmap's spacing. */
.masthead img {
display: block;
max-width: 100%;
height: auto;
margin: -1.7px 0 -1.1px;
}
.masthead .tagline {
background: #000;
color: #fff;
font-weight: bold;
/* A one-line label has no use for the body's prose leading. */
line-height: 1;
padding: .3rem .4rem;
margin-top: .4rem;
}
.wrap {
background: #ccd;
background: var(--panel);
border-bottom: 6px solid #000;
padding: 1.5rem;
/* The rings the 2007 pages carried. Transparent, so it rides on --panel. */
background-image: url(images/bg_rings.svg);
background-repeat: no-repeat;
background-position: top right;
}
@media (prefers-color-scheme: dark) {
/* Rings dropped, not inverted: their lavender is a light-panel tone. */
.wrap { background-image: none; }
/* The wordmark is black on transparent, and all but vanishes on the dark field. */
.masthead img { filter: invert(1); }
}
@media (min-width: 62rem) {
.wrap {
display: grid;
grid-template-columns: 14rem minmax(0, 1fr);
gap: 2.5rem;
align-items: start;
}
/* The hub is its own navigation, so it spans the full width. */
.wrap.solo { display: block; }
}
footer {
max-width: 68rem;
margin: .5rem auto 0;
color: #eef;
font-size: .8em;
}
@media (prefers-color-scheme: dark) {
footer { color: var(--ink-soft); }
}
/* ---- typography ---- */
h2, h3 { scroll-margin-top: 1rem; }
h1 { font-size: 1.9em; margin: 0 0 .3em; line-height: 1.2; }
h2 { font-size: 1.45em; margin: 2.5em 0 .5em; padding-bottom: .2em; border-bottom: 2px solid var(--rule); }
h3 { font-size: 1.15em; margin: 2em 0 .4em; }
h2:first-child, h1 + h2 { margin-top: 1em; }
p { margin: 0 0 1em; }
a { color: #00c; color: var(--link); }
a:hover { color: #c00; color: var(--link-hover); }
code, kbd, pre {
font-family: ui-monospace, "Cascadia Mono", Consolas, "DejaVu Sans Mono", monospace;
font-size: .9em;
}
code, kbd { background: var(--code-bg); padding: .1em .35em; border-radius: 3px; }
/* Legacy pages carry wide code samples: scroll them, never the page. */
pre {
overflow-x: auto;
max-width: 100%;
padding: .7rem;
background: var(--code-bg);
border-left: 3px solid var(--accent);
}
.lede { font-size: 1.08em; color: var(--ink-soft); }
hr { height: 1px; border: 0; background: var(--rule); margin: 2em 0; }
img { max-width: 100%; height: auto; }
/* Legacy prose carries bare URLs long enough to push a phone sideways. */
body { overflow-wrap: break-word; }
/* The reference pages carry tables wider than any column: scroll the table, not
the page. The guide's own MIME table is narrow and reads better left alone. */
main table:not(.mime) { display: block; overflow-x: auto; }
/* ---- table of contents ---- */
.toc { font-size: .92em; }
@media (min-width: 62rem) {
.toc { position: sticky; top: 1rem; max-height: calc(100vh - 2rem); overflow-y: auto; }
}
.toc h2 {
font-size: .8em;
text-transform: uppercase;
letter-spacing: .08em;
margin: 0 0 .5em;
border: 0;
color: var(--ink-soft);
}
.toc ol, .toc ul { list-style: none; margin: 0 0 1.5em; padding: 0; }
.toc li { margin: 0; }
.toc a {
display: block;
padding: .2rem .5rem;
text-decoration: none;
border-left: 3px solid transparent;
color: var(--ink);
}
.toc a:hover { background: var(--accent); }
.toc a[aria-current="true"] { border-left-color: var(--link); font-weight: bold; }
.toc .here { font-weight: bold; }
/* ---- callouts ---- */
.note {
margin: 0 0 1em;
padding: .7rem 1rem;
border-left: 4px solid var(--accent);
background: var(--chip);
font-size: .95em;
}
.note > :last-child { margin-bottom: 0; }
/* ---- the hub ---- */
.hub {
display: grid;
gap: 1.5rem;
margin: 2rem 0 0;
padding: 0;
list-style: none;
}
@media (min-width: 46rem) {
.hub { grid-template-columns: repeat(2, minmax(0, 1fr)); }
}
.hub > li {
padding: 1.2rem;
background: var(--chip);
border-radius: .5rem;
}
.hub h2 {
margin: 0 0 .8em;
font-size: 1.1em;
text-transform: uppercase;
letter-spacing: .06em;
border: 0;
color: var(--ink-soft);
}
.hub dl { margin: 0; }
.hub dt { margin: 0 0 .1em; font-weight: bold; }
.hub dd { margin: 0 0 1em; font-size: .93em; color: var(--ink-soft); }
.hub dd:last-child { margin-bottom: 0; }
/* Option names in the glosses are short; without this a browser breaks
"--changes" after its hyphens. */
.hub code { white-space: nowrap; }
.dated { font-weight: normal; font-size: .85em; color: var(--ink-soft); }
/* ---- content tables ---- */
/* The reference pages use bare <table> for their content tables. The guide's MIME
table is styled beside the guide and opts out. */
main table:not(.mime) {
border-collapse: collapse;
margin: 0 0 1em;
font-size: .92em;
}
main table:not(.mime) td, main table:not(.mime) th {
border: 1px solid var(--rule);
padding: .3em .6em;
text-align: left;
vertical-align: top;
}
main table:not(.mime) th, main table:not(.mime) tr.head td {
background: var(--accent);
font-weight: bold;
}
/* ---- screenshots ---- */
figure {
margin: 0 0 1.5em;
text-align: center;
}
figure img {
border: 1px solid var(--rule);
background: #fff;
box-shadow: 0 2px 8px var(--shadow);
cursor: zoom-in;
}
figcaption {
margin-top: .5em;
font-size: .88em;
color: var(--ink-soft);
}
/* Zoom overlay. Closed <dialog> is hidden by the UA, so nothing shows without JS. */
dialog#zoom {
max-width: 96vw;
max-height: 96vh;
padding: 0;
border: 0;
background: transparent;
}
dialog#zoom img {
display: block;
max-width: 96vw;
max-height: 96vh;
width: auto;
height: auto;
cursor: zoom-out;
}
dialog#zoom::backdrop { background: rgba(0, 0, 0, .8); }
/* ---- filter box (guide options, and the FAQ) ---- */
.filter {
display: flex;
gap: .5rem;
align-items: center;
margin: 0 0 1.5rem;
}
.filter input {
flex: 1 1 auto;
padding: .5rem .7rem;
border: 1px solid var(--rule);
border-radius: .3rem;
background: var(--panel);
color: var(--ink);
font: inherit;
}
.filter output { font-size: .88em; color: var(--ink-soft); white-space: nowrap; }
/* ---- misc ---- */
.skip {
position: absolute;
left: -9999px;
}
.skip:focus {
left: 1rem;
top: 1rem;
background: var(--panel);
padding: .5rem;
z-index: 10;
}
@media print {
body { background: #fff; }
.toc, .filter { display: none; }
.wrap { display: block; border: 0; }
}

104
html/doc.js
View File

@@ -1,104 +0,0 @@
/* Shared behaviour for the documentation pages: click an image to enlarge it, and
mark the section being read in the sidebar. Both are optional, and both are
written to fail quietly on an old browser rather than break the page: a Windows 7
machine may still open these files in Internet Explorer. */
(function () {
"use strict";
function each(list, fn) {
Array.prototype.forEach.call(list, fn);
}
/* ---- click to enlarge ---- */
function zoomable(node) {
while (node && node !== document.body) {
if (node.tagName === "IMG" && node.parentNode &&
node.parentNode.tagName === "FIGURE") {
return node;
}
node = node.parentNode;
}
return null;
}
function enlarge() {
var zoom = document.getElementById("zoom");
if (!zoom || typeof zoom.showModal !== "function") {
return;
}
var target = zoom.getElementsByTagName("img")[0];
document.addEventListener("click", function (event) {
var img = zoomable(event.target);
if (!img) {
return;
}
event.preventDefault();
target.src = img.src;
target.alt = img.alt;
zoom.showModal();
});
zoom.addEventListener("click", function () {
zoom.close();
});
}
/* ---- highlight the section being read ---- */
function highlight() {
var marks = [];
each(document.querySelectorAll(".toc a, .tabstrip a"), function (a) {
if (a.hash && a.hash.length > 1) {
marks.push(a);
}
});
if (!marks.length || !window.IntersectionObserver) {
return;
}
var byId = {};
each(marks, function (a) {
var id = a.hash.slice(1);
(byId[id] = byId[id] || []).push(a);
});
var visible = {};
var observer = new IntersectionObserver(function (entries) {
each(entries, function (entry) {
visible[entry.target.id] = entry.isIntersecting;
});
var top = Object.keys(byId).filter(function (id) {
return visible[id];
})[0];
each(marks, function (a) {
if (a.getAttribute("aria-current") === "true") {
a.removeAttribute("aria-current");
}
});
if (top) {
each(byId[top], function (a) {
a.setAttribute("aria-current", "true");
});
}
}, { rootMargin: "-10% 0px -70% 0px" });
Object.keys(byId).forEach(function (id) {
var el = document.getElementById(id);
if (el) {
observer.observe(el);
}
});
}
function start() {
enlarge();
highlight();
}
if (document.readyState === "loading") {
document.addEventListener("DOMContentLoaded", start);
} else {
start();
}
})();

Some files were not shown because too many files have changed in this diff Show More