mirror of
https://github.com/xroche/httrack.git
synced 2026-08-02 14:01:43 +03:00
Compare commits
385 Commits
3.49.8
...
fix-915-st
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e559a608b9 | ||
|
|
f3c3bf9aec | ||
|
|
2d041866ed | ||
|
|
36de4ca77f | ||
|
|
b673de16ac | ||
|
|
62a5ee7f2e | ||
|
|
cbcafb8e34 | ||
|
|
0b97f9632d | ||
|
|
82fe2b5008 | ||
|
|
819e609dbe | ||
|
|
5f52c3d942 | ||
|
|
d07839201c | ||
|
|
218403ed81 | ||
|
|
592f3dd154 | ||
|
|
6cc4f704f1 | ||
|
|
620418a9c6 | ||
|
|
0ed5c6b4ea | ||
|
|
150ddde375 | ||
|
|
f3fa3a8b98 | ||
|
|
311b99afd4 | ||
|
|
ef81f5b488 | ||
|
|
22c506944d | ||
|
|
b127323a68 | ||
|
|
fac60dff4a | ||
|
|
264478c846 | ||
|
|
8f383f4caf | ||
|
|
46294200ed | ||
|
|
3414596efe | ||
|
|
3c96b0f209 | ||
|
|
f1ace40d47 | ||
|
|
6b9740dbc2 | ||
|
|
3e480bf990 | ||
|
|
401323ff7e | ||
|
|
a21f84ee37 | ||
|
|
8d51e11c6e | ||
|
|
51b0142599 | ||
|
|
dc28c80ad8 | ||
|
|
21de744be6 | ||
|
|
3e4b131456 | ||
|
|
3dac180aac | ||
|
|
21d9289945 | ||
|
|
c4879cd25c | ||
|
|
cd3051ca1a | ||
|
|
16bbf2ad27 | ||
|
|
ed0906a4d4 | ||
|
|
b3cba2ba5a | ||
|
|
90b98e740e | ||
|
|
836c630f34 | ||
|
|
f0cd4b5089 | ||
|
|
22e9520636 | ||
|
|
a0a2bee019 | ||
|
|
457682b5df | ||
|
|
91d12d4040 | ||
|
|
5c0918f328 | ||
|
|
be9d9927a6 | ||
|
|
97d7da5f19 | ||
|
|
15c66557b9 | ||
|
|
db4d429468 | ||
|
|
1b53b43176 | ||
|
|
2e67587ec7 | ||
|
|
45557b3dd6 | ||
|
|
63af0dc4a2 | ||
|
|
e5c0918d5d | ||
|
|
55dbca8beb | ||
|
|
8c5b208fac | ||
|
|
b7cff7aeee | ||
|
|
bc8b6cfa00 | ||
|
|
260cef25fe | ||
|
|
bd60dac914 | ||
|
|
0984aa2530 | ||
|
|
bc6c53598c | ||
|
|
809d5b6ffc | ||
|
|
536d7515ed | ||
|
|
374087a81c | ||
|
|
83079b788d | ||
|
|
1fa3cb4e74 | ||
|
|
bab4b70a71 | ||
|
|
dcdcc8745c | ||
|
|
c65d1c43c7 | ||
|
|
95f8ebaa44 | ||
|
|
3a096c589a | ||
|
|
1a4a5082b7 | ||
|
|
4fd6767a8f | ||
|
|
56389103bf | ||
|
|
a4db58f7b7 | ||
|
|
1f9728d816 | ||
|
|
99bd9bb674 | ||
|
|
29dfd2df59 | ||
|
|
55765815f5 | ||
|
|
869b8479e9 | ||
|
|
9fe47c3986 | ||
|
|
37fa549ac5 | ||
|
|
de8c0eebfc | ||
|
|
9e29c1e159 | ||
|
|
9571fb9a6a | ||
|
|
3e8595c46f | ||
|
|
0774d47d2f | ||
|
|
7e2209d4cc | ||
|
|
9484b32ecd | ||
|
|
1d647bfecd | ||
|
|
b2dc012263 | ||
|
|
aa1131982b | ||
|
|
f72e7ebe96 | ||
|
|
3b53bf85f2 | ||
|
|
ce7dcfa9de | ||
|
|
069573edc3 | ||
|
|
a75f437df9 | ||
|
|
783f6ee1f5 | ||
|
|
913caf68be | ||
|
|
59660102d6 | ||
|
|
e96399910b | ||
|
|
52d0ab2356 | ||
|
|
ca533abefd | ||
|
|
dc6819b7b7 | ||
|
|
3247e5b183 | ||
|
|
bef7120423 | ||
|
|
7cf54af485 | ||
|
|
c32a47110e | ||
|
|
1027a9f392 | ||
|
|
d0a1573a04 | ||
|
|
4362ac34cb | ||
|
|
6579436607 | ||
|
|
cb98b1b197 | ||
|
|
da8fbfff49 | ||
|
|
a7fbd3f739 | ||
|
|
c4b803eb33 | ||
|
|
408316db99 | ||
|
|
450db10d92 | ||
|
|
dfdf10e7fd | ||
|
|
7fe073e9fb | ||
|
|
f47359247d | ||
|
|
dfc72fdbc6 | ||
|
|
8ed607b078 | ||
|
|
0708bf9f98 | ||
|
|
301d7717be | ||
|
|
e13f0f10c9 | ||
|
|
3265a5de00 | ||
|
|
d33f666b38 | ||
|
|
6223739cba | ||
|
|
301f5c2f2f | ||
|
|
dd321171b4 | ||
|
|
2f158c05d0 | ||
|
|
6c74d94802 | ||
|
|
00b0f5728c | ||
|
|
93314bcef9 | ||
|
|
d3d3bce8af | ||
|
|
b3e51d753b | ||
|
|
5099efc1cf | ||
|
|
e1d4c35ee6 | ||
|
|
69c562bf0c | ||
|
|
d83ed3fdee | ||
|
|
ce278a4184 | ||
|
|
f6f46e88b2 | ||
|
|
1e0c009273 | ||
|
|
f894a64ff8 | ||
|
|
febdd08cae | ||
|
|
3e1d1abdab | ||
|
|
8f7bfbb2f6 | ||
|
|
2ba353eb6f | ||
|
|
3dd8a97611 | ||
|
|
84656760be | ||
|
|
f26d11d3aa | ||
|
|
46d65c6610 | ||
|
|
d2e94b1c99 | ||
|
|
91071cb003 | ||
|
|
2b5e740b55 | ||
|
|
1d62527d2d | ||
|
|
0d25999069 | ||
|
|
8904656164 | ||
|
|
86b46947fb | ||
|
|
71c1764525 | ||
|
|
db31be5a6c | ||
|
|
5100142e03 | ||
|
|
1cab56bfe4 | ||
|
|
1d0e2222d4 | ||
|
|
4166200465 | ||
|
|
2382ca3aa0 | ||
|
|
68a9a247d6 | ||
|
|
86bc02d8da | ||
|
|
e675d60301 | ||
|
|
1c8b93d9e7 | ||
|
|
d267aaf1bf | ||
|
|
226702fcab | ||
|
|
491774abda | ||
|
|
dbad05fcf0 | ||
|
|
db19089448 | ||
|
|
eb9e353f0b | ||
|
|
8a8fd7f9c0 | ||
|
|
1c1c0b1267 | ||
|
|
fe685d4117 | ||
|
|
f12d5e7434 | ||
|
|
e01e1b34de | ||
|
|
a67eb57def | ||
|
|
6634d48c21 | ||
|
|
a2c75b5316 | ||
|
|
6aabb3ba09 | ||
|
|
088f0711b8 | ||
|
|
ce481efeca | ||
|
|
7b68d7dc96 | ||
|
|
a47b7bee4d | ||
|
|
7be61d3111 | ||
|
|
3cc49b79c3 | ||
|
|
959deb0afb | ||
|
|
18bdc24d15 | ||
|
|
0b772ec6ba | ||
|
|
a0f2f89ec3 | ||
|
|
93ec06f404 | ||
|
|
bc67e74dea | ||
|
|
c7fcdebe46 | ||
|
|
eb46d8d469 | ||
|
|
9bc53d40d2 | ||
|
|
938a873e19 | ||
|
|
ed05faaaa8 | ||
|
|
5e56a913a5 | ||
|
|
29e868b2ca | ||
|
|
f1418bac3c | ||
|
|
092a557305 | ||
|
|
cfe7c7e68f | ||
|
|
89b1e485d7 | ||
|
|
08481464bd | ||
|
|
3e7b9f91ee | ||
|
|
800d498893 | ||
|
|
991d259981 | ||
|
|
d1bf4f7a25 | ||
|
|
86e145fc1c | ||
|
|
d9ff780d0f | ||
|
|
b026c80bb6 | ||
|
|
e0648beff9 | ||
|
|
ee1aebd935 | ||
|
|
b4957f0625 | ||
|
|
c9d36c82c7 | ||
|
|
1fa0184468 | ||
|
|
79e46eeceb | ||
|
|
436ac53793 | ||
|
|
bc4b2c7b8e | ||
|
|
4fafe283b4 | ||
|
|
9ec5a3a067 | ||
|
|
cd38e6e68d | ||
|
|
e692e56592 | ||
|
|
7d9d4ed890 | ||
|
|
573167e8dd | ||
|
|
e26a5f5c6a | ||
|
|
627ae2b043 | ||
|
|
71ab3574ef | ||
|
|
076127ddea | ||
|
|
dced0208e1 | ||
|
|
6e34c30a7c | ||
|
|
eac13ea248 | ||
|
|
dc3e485f92 | ||
|
|
fac84f1631 | ||
|
|
c32d286938 | ||
|
|
5a7ab7a653 | ||
|
|
5756a9830c | ||
|
|
28c11d55eb | ||
|
|
b5f28f6eb5 | ||
|
|
3d65bb2e3b | ||
|
|
68897b09ce | ||
|
|
e8d1bf9c19 | ||
|
|
ef40277c24 | ||
|
|
797acecfd2 | ||
|
|
8237c3ed2b | ||
|
|
75b815e1fb | ||
|
|
8d02e60907 | ||
|
|
e70607e70c | ||
|
|
cada6a1ae3 | ||
|
|
145ea4769f | ||
|
|
c5da5a2f2e | ||
|
|
6eaefe6233 | ||
|
|
33249be5a0 | ||
|
|
167ede32c0 | ||
|
|
c8b7c63c9b | ||
|
|
04f37b2094 | ||
|
|
c07b19bc3d | ||
|
|
04bbb489cf | ||
|
|
a24b1d3c9f | ||
|
|
1abf867333 | ||
|
|
771d11326e | ||
|
|
7faceaf7d2 | ||
|
|
9105732112 | ||
|
|
9c01812141 | ||
|
|
7a02d5e411 | ||
|
|
3c7e9fa46d | ||
|
|
a707d4b845 | ||
|
|
65c1016a4e | ||
|
|
fd2252dc8c | ||
|
|
53a257448f | ||
|
|
8ba3cb6c03 | ||
|
|
b9afe755bd | ||
|
|
407916e3f6 | ||
|
|
15021cd470 | ||
|
|
5f7045c3a4 | ||
|
|
f66ec195aa | ||
|
|
7928db5729 | ||
|
|
20be48d56c | ||
|
|
484fc47eab | ||
|
|
abaf9b69a2 | ||
|
|
b0466b1d7b | ||
|
|
f785286c87 | ||
|
|
440a8603a9 | ||
|
|
4979e58dc0 | ||
|
|
894cf5a8d2 | ||
|
|
0c1aa51385 | ||
|
|
fb4267c6d7 | ||
|
|
f0b044c2f3 | ||
|
|
dfafe28002 | ||
|
|
a3f04bde72 | ||
|
|
11beef52e1 | ||
|
|
d7c4eab1f5 | ||
|
|
2eac19655b | ||
|
|
83c231d50e | ||
|
|
9d29b8329b | ||
|
|
ac4a1ca48e | ||
|
|
9f2f2e52fa | ||
|
|
92db2f2b41 | ||
|
|
ec52112446 | ||
|
|
1eaddc9c0e | ||
|
|
d97a7bdfd9 | ||
|
|
d2d02d87c2 | ||
|
|
4958bb8666 | ||
|
|
07da404cb8 | ||
|
|
694e45c698 | ||
|
|
db9ec2cc3b | ||
|
|
6a9ab2a11f | ||
|
|
13b31986d5 | ||
|
|
bd7e0989f6 | ||
|
|
bd74ec7cab | ||
|
|
1ed8ffad64 | ||
|
|
b68de172fa | ||
|
|
aabfd34380 | ||
|
|
65ff9e0f11 | ||
|
|
730a1c8c5b | ||
|
|
f9ee4702a2 | ||
|
|
cca83e5f4a | ||
|
|
97f398e508 | ||
|
|
a62f93a107 | ||
|
|
799ec88dc7 | ||
|
|
71af4a24f0 | ||
|
|
e17f4f12a0 | ||
|
|
5be8ba4bbd | ||
|
|
247a46068e | ||
|
|
669947cd23 | ||
|
|
40a66600ff | ||
|
|
768756e231 | ||
|
|
b138c87a93 | ||
|
|
3de47433b7 | ||
|
|
fb8827718e | ||
|
|
7228210061 | ||
|
|
38882c0aee | ||
|
|
bfc4a016ab | ||
|
|
756d8fb8bd | ||
|
|
5501faa7b1 | ||
|
|
6322b6fb1f | ||
|
|
58f368a91a | ||
|
|
c97b3e233e | ||
|
|
b615a4e7fd | ||
|
|
594cf0da39 | ||
|
|
3845cd1fb3 | ||
|
|
94bffb0804 | ||
|
|
a5c86e7e89 | ||
|
|
54f5717057 | ||
|
|
40fc9de360 | ||
|
|
4614eefefe | ||
|
|
b0e8262db0 | ||
|
|
addbd3136b | ||
|
|
a64c4cd160 | ||
|
|
1611dbcabf | ||
|
|
099501ee50 | ||
|
|
1b9eefa3b4 | ||
|
|
9c8d3a41eb | ||
|
|
ae77cd9d6d | ||
|
|
51b8dcd81c | ||
|
|
bcce664143 | ||
|
|
7a24add87c | ||
|
|
2308e7bafd | ||
|
|
ef5691fc47 | ||
|
|
0a6eb73903 | ||
|
|
fdb243e5a2 | ||
|
|
f8546e146d | ||
|
|
b7f602f2eb | ||
|
|
550100b56a | ||
|
|
33ddb27243 | ||
|
|
4606dfbf66 | ||
|
|
a6f1b9a3dd | ||
|
|
fb35d6a0f1 | ||
|
|
8a270fec03 |
5
.flake8
Normal file
5
.flake8
Normal file
@@ -0,0 +1,5 @@
|
||||
[flake8]
|
||||
# Match black's formatting so the two tools don't fight.
|
||||
max-line-length = 88
|
||||
# E203/W503 conflict with black's slice and line-break style.
|
||||
extend-ignore = E203, W503
|
||||
11
.gitattributes
vendored
Normal file
11
.gitattributes
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
# Resource scripts are Windows tooling input and must stay CRLF: the engine has no
|
||||
# encoding guard, so an autocrlf checkout could otherwise flatten them silently.
|
||||
*.rc text eol=crlf
|
||||
|
||||
# The test scripts run under Git Bash on the Windows CI; a CRLF checkout makes
|
||||
# bash die on $'\r' on every line of them.
|
||||
*.test text eol=lf
|
||||
*.sh text eol=lf
|
||||
|
||||
# Scoped to this file so a union can never touch build logic.
|
||||
tests/tests-list.mk merge=union
|
||||
20
.github/dependabot.yml
vendored
Normal file
20
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Bumps builtin-baseline in src/vcpkg.json so the pinned OpenSSL/zlib can't rot.
|
||||
# windows-build.yml validates each bump; the pin makes what ships reproducible.
|
||||
- package-ecosystem: vcpkg
|
||||
directory: /src
|
||||
schedule:
|
||||
interval: weekly
|
||||
|
||||
# Keep the workflow action pins current (they only rot manually otherwise).
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
|
||||
# src/coucal is a hard build dependency, and nothing else notices when it moves.
|
||||
- package-ecosystem: gitsubmodule
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
490
.github/workflows/ci.yml
vendored
490
.github/workflows/ci.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
env:
|
||||
CC: ${{ matrix.cc }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
@@ -41,21 +41,97 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential clang autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: Configure
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Regenerate: configure and the Makefile.in's are not tracked.
|
||||
autoreconf -fi
|
||||
# Disabling zlib must fail here rather than at link with a pile of
|
||||
# undefined minizip references (#735). Both spellings, so a rewrite
|
||||
# cannot keep one and lose the other. Probed out-of-tree to leave
|
||||
# nothing behind.
|
||||
nozlib="$RUNNER_TEMP/nozlib"
|
||||
for arg in --without-zlib --with-zlib=no; do
|
||||
rm -rf "$nozlib" && mkdir -p "$nozlib"
|
||||
if (cd "$nozlib" && "$GITHUB_WORKSPACE/configure" "$arg" >out.log 2>&1); then
|
||||
echo "::error::configure $arg succeeded; it must be rejected"
|
||||
exit 1
|
||||
fi
|
||||
# ... and for the stated reason, not an unrelated configure failure.
|
||||
grep -q "zlib cannot be disabled" "$nozlib/out.log" \
|
||||
|| { cat "$nozlib/out.log"; exit 1; }
|
||||
done
|
||||
./configure
|
||||
# Same dead end from the compile side. The bare compile is the
|
||||
# control: without it a broken probe would pass vacuously.
|
||||
hdr='#include "htsglobal.h"'
|
||||
echo "$hdr" | $CC -I. -Isrc -fsyntax-only -xc -
|
||||
if echo "$hdr" | $CC -DHTS_USEZLIB=0 -I. -Isrc -fsyntax-only -xc - 2>/dev/null; then
|
||||
echo "::error::-DHTS_USEZLIB=0 compiled; htsglobal.h must reject it"
|
||||
exit 1
|
||||
fi
|
||||
# a missing decoder would silently drop the coding from Accept-Encoding
|
||||
grep -q "define HTS_USEBROTLI 1" config.h
|
||||
grep -q "define HTS_USEZSTD 1" config.h
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
# A backstop only: tests/test-timeout.sh bounds each test, and a healthy run
|
||||
# is a minute here, two on macOS. Without it a stall ran to the job's 6h default.
|
||||
- name: Test
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# Reproduce the Debian buildds: they build in a minimal chroot with no
|
||||
# python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
|
||||
# runners ship python3, so every other job hides this path; here we remove it
|
||||
# before `make check`. This is the guard that would have caught the 3.49.10-1
|
||||
# FTBFS (28_local-pause failed instead of skipping when python3 was absent).
|
||||
buildd-no-python3:
|
||||
name: build (no python3, Debian buildd)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: Configure
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Regenerate from configure.ac/Makefile.am to validate them; the
|
||||
# committed generated files already let a plain checkout build.
|
||||
autoreconf -fi
|
||||
./configure
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: make check
|
||||
- name: Test without python3
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Hide every python3* so `command -v python3` fails like it does in the
|
||||
# buildd chroot; masking with /bin/false would still resolve.
|
||||
sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
|
||||
-exec mv {} {}.hidden \;
|
||||
! command -v python3
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -67,34 +143,169 @@ jobs:
|
||||
# so point configure at it; everything else is in the SDK or default paths.
|
||||
macos:
|
||||
name: build (macOS arm64, clang)
|
||||
runs-on: macos-14
|
||||
runs-on: macos-15
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
brew install autoconf automake libtool autoconf-archive
|
||||
brew install autoconf automake libtool autoconf-archive brotli zstd
|
||||
|
||||
- name: Configure
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ssl="$(brew --prefix openssl@3)"
|
||||
brewp="$(brew --prefix)"
|
||||
autoreconf -fi
|
||||
./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib"
|
||||
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
|
||||
LDFLAGS="-L${ssl}/lib -L${brewp}/lib"
|
||||
grep -q "define HTS_USEBROTLI 1" config.h
|
||||
grep -q "define HTS_USEZSTD 1" config.h
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(sysctl -n hw.ncpu)"
|
||||
|
||||
- name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
|
||||
# 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
|
||||
# instantly like Linux; alias them onto lo0 so they don't stall to timeout.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo ifconfig lo0 alias 127.0.0.2 up
|
||||
sudo ifconfig lo0 alias 127.0.0.3 up
|
||||
|
||||
- name: Test
|
||||
run: make check
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
|
||||
# browser path (open -W) and nothing else exercises htsserver. Install into a
|
||||
# temp prefix, then check webhttrack brings up htsserver and serves the UI.
|
||||
webhttrack-macos:
|
||||
name: webhttrack smoke (macOS arm64)
|
||||
runs-on: macos-15
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
brew install autoconf automake libtool autoconf-archive brotli zstd
|
||||
|
||||
- name: Build and install into a temp prefix
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ssl="$(brew --prefix openssl@3)"
|
||||
brewp="$(brew --prefix)"
|
||||
autoreconf -fi
|
||||
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
|
||||
LDFLAGS="-L${ssl}/lib -L${brewp}/lib" \
|
||||
--prefix="$RUNNER_TEMP/inst"
|
||||
make -j"$(sysctl -n hw.ncpu)"
|
||||
make install
|
||||
|
||||
- name: Smoke-test webhttrack
|
||||
run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
|
||||
|
||||
# Same smoke on the platform that ships webhttrack as a package: the install layout
|
||||
# it asserts (a relative data symlink, #885) is what Debian consumes, and the macOS
|
||||
# job alone left that untested on Linux.
|
||||
webhttrack-linux:
|
||||
name: webhttrack smoke (Linux x86-64)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: Build and install into a temp prefix
|
||||
run: |
|
||||
set -euo pipefail
|
||||
./bootstrap
|
||||
./configure --prefix="$RUNNER_TEMP/inst"
|
||||
make -j"$(nproc)"
|
||||
make install
|
||||
|
||||
- name: Smoke-test webhttrack
|
||||
run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
|
||||
|
||||
# The macOS app bundle (#886): assemble it, then prove it still works after being
|
||||
# moved, which is the only thing a .app has to survive that a prefix install does not.
|
||||
macos-app:
|
||||
name: macOS app bundle (arm64)
|
||||
runs-on: macos-15
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
brew install autoconf automake libtool autoconf-archive brotli zstd
|
||||
|
||||
# --disable-shared keeps libhttrack inside the binaries, so the bundle carries no
|
||||
# absolute dylib path back to the staging prefix.
|
||||
- name: Build and install into a temp prefix
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ssl="$(brew --prefix openssl@3)"
|
||||
brewp="$(brew --prefix)"
|
||||
./bootstrap
|
||||
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
|
||||
LDFLAGS="-L${ssl}/lib -L${brewp}/lib" \
|
||||
--prefix="$RUNNER_TEMP/inst" --disable-shared
|
||||
make -j"$(sysctl -n hw.ncpu)"
|
||||
make install
|
||||
|
||||
- name: Assemble HTTrack.app
|
||||
run: |
|
||||
set -euo pipefail
|
||||
make macos-app APP_FLAGS="--prefix $RUNNER_TEMP/inst --out $RUNNER_TEMP"
|
||||
plutil -lint "$RUNNER_TEMP/HTTrack.app/Contents/Info.plist"
|
||||
|
||||
- name: Run it through the bundle stub
|
||||
run: |
|
||||
set -euo pipefail
|
||||
app="$RUNNER_TEMP/HTTrack.app"
|
||||
bash tests/webhttrack-smoke.sh "$app/Contents/Resources" "$app/Contents/MacOS/HTTrack"
|
||||
|
||||
- name: Run it again after moving it
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$RUNNER_TEMP/moved"
|
||||
mv "$RUNNER_TEMP/HTTrack.app" "$RUNNER_TEMP/moved/"
|
||||
rm -rf "$RUNNER_TEMP/inst"
|
||||
app="$RUNNER_TEMP/moved/HTTrack.app"
|
||||
bash tests/webhttrack-smoke.sh "$app/Contents/Resources" "$app/Contents/MacOS/HTTrack"
|
||||
|
||||
# No Developer ID here, so Gatekeeper cannot be exercised; an ad-hoc signature
|
||||
# still proves the bundle is well formed enough to sign.
|
||||
- name: Check the bundle is signable
|
||||
run: |
|
||||
set -euo pipefail
|
||||
app="$RUNNER_TEMP/moved/HTTrack.app"
|
||||
codesign -s - --force --deep "$app"
|
||||
codesign --verify --strict --verbose=2 "$app"
|
||||
|
||||
# Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
|
||||
# -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
|
||||
# and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
|
||||
@@ -104,7 +315,7 @@ jobs:
|
||||
name: build (linux i386, gcc -m32)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
@@ -115,7 +326,8 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential gcc-multilib autoconf automake libtool \
|
||||
autoconf-archive zlib1g-dev:i386 libssl-dev:i386
|
||||
autoconf-archive zlib1g-dev:i386 libssl-dev:i386 \
|
||||
libbrotli-dev:i386 libzstd-dev:i386
|
||||
|
||||
- name: Configure
|
||||
run: |
|
||||
@@ -127,7 +339,10 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: make check
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -144,7 +359,7 @@ jobs:
|
||||
name: sanitize (ASan+UBSan, gcc)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
@@ -154,7 +369,7 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: Configure (sanitized)
|
||||
run: |
|
||||
@@ -182,12 +397,102 @@ jobs:
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
run: make check
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# MemorySanitizer catches reads of uninitialized memory (#143's stack-garbage
|
||||
# size filter) that ASan/UBSan miss. It flags any byte an uninstrumented lib
|
||||
# wrote, so the job stays in our own code: offline self-tests only, no openssl
|
||||
# (--disable-https), no zlib cache tests, static (the runtime is not in .so's).
|
||||
msan:
|
||||
name: msan (MemorySanitizer, clang)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential clang autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev
|
||||
|
||||
- name: Configure (MSan, static, no https)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure CC=clang \
|
||||
CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
|
||||
LDFLAGS="-fsanitize=memory" \
|
||||
--disable-https --disable-shared --enable-static
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test (offline self-tests under MSan)
|
||||
env:
|
||||
MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
|
||||
# skipped here (uninstrumented libz floods MSan with false positives).
|
||||
tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs" TESTS="$tests"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
|
||||
# parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
|
||||
# (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
|
||||
# catastrophic backtracking -- and pins the regression seeds for the bugs the
|
||||
# fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
|
||||
fuzz:
|
||||
name: fuzz (libFuzzer corpus replay, clang)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential clang autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: Configure (fuzzers, static)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure CC=clang \
|
||||
CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
|
||||
LDFLAGS="-fsanitize=address,undefined" \
|
||||
--enable-fuzzers --disable-shared
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Replay corpora
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
run: bash fuzz/run-fuzzers.sh fuzz check
|
||||
|
||||
# Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
|
||||
# the configuration users on minimal systems build, and one libssl is not even
|
||||
# installed here so configure cannot silently re-enable it. The matrix above
|
||||
@@ -197,7 +502,7 @@ jobs:
|
||||
name: build (no openssl, --disable-https)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
@@ -218,43 +523,89 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: make check
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# Validate the Debian packaging via the same script maintainers release with.
|
||||
# One amd64/gcc run is enough: packaging (control/rules/manifest/lintian/quilt
|
||||
# source build) is arch- and compiler-independent, and the build matrix above
|
||||
# already covers compile portability. lintian runs with --fail-on=error.
|
||||
deb:
|
||||
name: deb package (lintian)
|
||||
# OpenSSL 4.0 removes the API 3.x deprecated. Building with that set hidden
|
||||
# keeps the engine ready for it: a deprecated call fails here rather than the
|
||||
# day a distro moves. Compile-only, since the headers are what is under test.
|
||||
openssl-no-deprecated:
|
||||
name: build (OpenSSL, no deprecated API)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install packaging toolchain
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev \
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: Configure (deprecated OpenSSL API hidden)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure CPPFLAGS=-DOPENSSL_NO_DEPRECATED
|
||||
# A build with https off would pass this job vacuously.
|
||||
grep -q "define HTS_USEOPENSSL 1" config.h
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
# Validate the Debian packaging via the same script maintainers release with.
|
||||
# One amd64/gcc run is enough: packaging (control/rules/manifest/lintian/quilt
|
||||
# source build) is arch- and compiler-independent, and the build matrix above
|
||||
# already covers compile portability. mkdeb.sh runs lintian as an explicit gate
|
||||
# (debuild does not propagate lintian's exit) with --fail-on=error,warning.
|
||||
deb:
|
||||
name: deb package (lintian)
|
||||
runs-on: ubuntu-24.04
|
||||
# Build and gate inside Debian sid, the upload target. A Debian dpkg-deb
|
||||
# produces archive-legal xz members (an Ubuntu host defaults to zstd, which
|
||||
# the archive's lintian rejects), and sid's lintian carries the same
|
||||
# data-driven checks (embedded-lib fingerprints and the like) the buildds and
|
||||
# UDD apply -- so issues surface here instead of after upload.
|
||||
container: debian:sid
|
||||
steps:
|
||||
- name: Install packaging toolchain
|
||||
run: |
|
||||
set -euo pipefail
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates git \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev \
|
||||
debhelper devscripts lintian fakeroot
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# --unsigned: CI has no GPG key (also skips the release sig/checksums).
|
||||
# debuild builds every package, then lintian gates on errors.
|
||||
# mkdeb builds every package then runs the lintian gate (--fail-on=error,
|
||||
# warning); debuild runs the packaged test pass.
|
||||
#
|
||||
# DEB_BUILD_OPTIONS trims work CI does not need (release builds via
|
||||
# mkdeb.sh are untouched): noautodbgsym drops the -dbgsym packages whose
|
||||
# LTO payloads are slow to compress and that CI never ships; parallel uses
|
||||
# every core. We let debuild run its test pass -- the only one now that
|
||||
# mkdeb no longer runs its own -- so CI exercises the packaged tests.
|
||||
- name: Build Debian packages
|
||||
# every core.
|
||||
- name: Build and lint Debian packages
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The workspace volume is owned by the host runner uid, but the
|
||||
# container runs as root, so mkdeb's git calls (superproject and the
|
||||
# coucal submodule) trip "dubious ownership"; mark them all safe.
|
||||
git config --global --add safe.directory "*"
|
||||
export DEB_BUILD_OPTIONS="noautodbgsym parallel=$(nproc)"
|
||||
bash tools/mkdeb.sh --unsigned --no-release-artifacts
|
||||
|
||||
@@ -267,7 +618,7 @@ jobs:
|
||||
name: distcheck (release tarball)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
@@ -277,7 +628,7 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: distcheck
|
||||
run: |
|
||||
@@ -292,7 +643,7 @@ jobs:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -331,12 +682,13 @@ jobs:
|
||||
man/makeman.sh
|
||||
src/htsbasiccharsets.sh
|
||||
src/htsentities.sh
|
||||
src/webhttrack
|
||||
src/webhttrack.in
|
||||
tests/*.sh
|
||||
tests/*.test
|
||||
tools/macos-app.sh
|
||||
tools/mkdeb.sh
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install linters
|
||||
run: |
|
||||
@@ -353,6 +705,16 @@ jobs:
|
||||
- name: shfmt
|
||||
run: shfmt -d -i 4 $SHELL_SCRIPTS
|
||||
|
||||
# MSBuild rejects a malformed .vcxproj with a bare MSB4025 and no build, so
|
||||
# catch it here in seconds rather than on a Windows runner minutes in.
|
||||
- name: XML well-formedness (MSBuild project files)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for f in src/*.vcxproj; do
|
||||
python3 -c "import sys,xml.dom.minidom; xml.dom.minidom.parse(sys.argv[1])" "$f"
|
||||
echo "ok $f"
|
||||
done
|
||||
|
||||
# Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
|
||||
# (it was shaped by an old Visual Studio formatter) and does not round-trip,
|
||||
# so we never reformat the whole tree -- only the lines a PR touches.
|
||||
@@ -361,7 +723,7 @@ jobs:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -385,7 +747,12 @@ jobs:
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
|
||||
base="origin/${{ github.base_ref }}"
|
||||
# Merge base, not the branch tip: master moves during a run, and a
|
||||
# tip-relative diff blames its new C on this PR.
|
||||
if ! base="$(git merge-base "origin/${{ github.base_ref }}" HEAD)"; then
|
||||
echo "::error::no merge base with origin/${{ github.base_ref }}; cannot scope the check."
|
||||
exit 1
|
||||
fi
|
||||
set +e
|
||||
diff="$(git clang-format --binary clang-format-19 --style=file \
|
||||
--diff --extensions c,h "$base")"
|
||||
@@ -407,3 +774,48 @@ jobs:
|
||||
echo "Fix locally with: git clang-format --binary clang-format-19 $base"
|
||||
exit 1 ;;
|
||||
esac
|
||||
|
||||
docs:
|
||||
name: documentation (chrome in sync, links resolve)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
# The pages carry their navigation as real markup, because they are read from
|
||||
# disk as often as over http and file:// has no includes. Regenerating has to
|
||||
# be a no-op, or a page has drifted from tools/doc-chrome.py.
|
||||
- name: Shared chrome is in sync
|
||||
run: python3 tools/doc-chrome.py --check
|
||||
|
||||
- name: Relative links and anchors resolve
|
||||
run: python3 tools/doc-links.py html
|
||||
|
||||
man-page-sync:
|
||||
name: man page / html in sync
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# html/httrack.man.html is groff-rendered from man/httrack.1 and committed.
|
||||
# Rendering needs the full groff html device, so CI can't regenerate it;
|
||||
# instead require the two to move together: a PR that touches httrack.1
|
||||
# must also touch the html, catching the "regenerated roff, forgot html".
|
||||
- name: httrack.1 changes must include html/httrack.man.html
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
|
||||
base="origin/${{ github.base_ref }}"
|
||||
# Three dots: merge-base scoped, so commits landing on master mid-run
|
||||
# are not counted as this PR's.
|
||||
changed="$(git diff --name-only "$base"...HEAD)"
|
||||
has() { printf '%s\n' "$changed" | grep -qx "$1"; }
|
||||
if has man/httrack.1 && ! has html/httrack.man.html; then
|
||||
echo "::error::man/httrack.1 changed but html/httrack.man.html did not."
|
||||
echo "Regenerate it with: make -C man regen-man-html (needs the full groff package)."
|
||||
exit 1
|
||||
fi
|
||||
echo "man/html sync OK."
|
||||
|
||||
72
.github/workflows/codeql.yml
vendored
Normal file
72
.github/workflows/codeql.yml
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
# CodeQL static analysis (C). The security-extended suite covers the classes
|
||||
# this codebase actually fights: overflows, tainted-size allocs, format bugs.
|
||||
name: CodeQL
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
schedule:
|
||||
# Weekly re-scan of master so new/updated queries land without a push.
|
||||
- cron: "17 4 * * 1"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: codeql-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: analyze (c-cpp)
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
# Upload findings to the repo's code-scanning dashboard.
|
||||
security-events: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev
|
||||
|
||||
- uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: c-cpp
|
||||
build-mode: manual
|
||||
queries: security-extended
|
||||
# fopen's umask-controlled 0666 is intended for mirror/cache/log
|
||||
# output; the one credential file (cookies.txt) is kept 0600 on Unix.
|
||||
config: |
|
||||
paths-ignore:
|
||||
# Demo callback samples, not part of libhttrack.
|
||||
- libtest
|
||||
# ProxyTrack: a separate legacy binary with no auth surface; its
|
||||
# recv/cache-parse code trips taint queries by design.
|
||||
- src/proxy
|
||||
query-filters:
|
||||
- exclude:
|
||||
id: cpp/world-writable-file-creation
|
||||
# Models auth-bypass-by-spoofing; httrack has no auth surface, its +/- crawl filter is a mirror boundary, not a security one.
|
||||
- exclude:
|
||||
id: cpp/user-controlled-bypass
|
||||
|
||||
# Manual build: CodeQL traces the compiler, so build exactly what ships.
|
||||
- name: Build
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure
|
||||
make -j"$(nproc)"
|
||||
|
||||
- uses: github/codeql-action/analyze@v4
|
||||
with:
|
||||
category: "/language:c-cpp"
|
||||
72
.github/workflows/screenshots.yml
vendored
Normal file
72
.github/workflows/screenshots.yml
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
# Documentation screenshots: drive the WebHTTrack UI and capture every screen.
|
||||
# Manual, and deliberately not part of the build gate -- run it after adding or
|
||||
# changing a UI page, and the artifact is the new set of shots.
|
||||
name: screenshots
|
||||
|
||||
on:
|
||||
# A push trigger as well as the button: workflow_dispatch is only offered for
|
||||
# workflows that already exist on the default branch.
|
||||
push:
|
||||
paths:
|
||||
- '.github/workflows/screenshots.yml'
|
||||
- 'tools/screenshot-walk.py'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
lang:
|
||||
description: 'lang.indexes number to render the UI in (1 = English)'
|
||||
default: '1'
|
||||
scale:
|
||||
description: 'device pixel ratio'
|
||||
default: '2'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
capture:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure
|
||||
make -j"$(nproc)" -C src htsserver
|
||||
|
||||
- name: Install the browser
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pip install --quiet playwright
|
||||
playwright install --with-deps chromium
|
||||
|
||||
- name: Capture
|
||||
env:
|
||||
LANGN: ${{ inputs.lang || '1' }}
|
||||
SCALE: ${{ inputs.scale || '2' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# --root is the dist root; the source tree already holds every piece
|
||||
# htsserver reads, so this needs no install.
|
||||
python3 tools/screenshot-walk.py --htsserver src/htsserver --root . \
|
||||
--lang "$LANGN" --scale "$SCALE" --out shots
|
||||
|
||||
- name: Upload the screens
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: screenshots
|
||||
path: shots
|
||||
if-no-files-found: error
|
||||
316
.github/workflows/windows-build.yml
vendored
Normal file
316
.github/workflows/windows-build.yml
vendored
Normal file
@@ -0,0 +1,316 @@
|
||||
# Windows build of the engine (VS2022 v143 + vcpkg): libhttrack, the CLI,
|
||||
# webhttrack, proxytrack.
|
||||
#
|
||||
# The autotools CI covers the unix builds; this covers the MSVC one, which had
|
||||
# no coverage at all and had drifted (the old .vcproj pinned OpenSSL 1.0.1j).
|
||||
# libhttrack.dll is what the WinHTTrack GUI links against.
|
||||
name: windows-build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
libhttrack:
|
||||
runs-on: windows-2022
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform: [x64, Win32]
|
||||
configuration: [Release]
|
||||
# Redirect vcpkg's default `files` binary cache into the workspace so
|
||||
# actions/cache can persist it. vcpkg builds openssl/brotli/zlib/zstd from
|
||||
# source otherwise, several minutes every run.
|
||||
env:
|
||||
VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\vcpkg_cache
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive # coucal lives in src/coucal
|
||||
|
||||
# Located through vswhere rather than microsoft/setup-msbuild: the repo
|
||||
# only allows GitHub-owned actions.
|
||||
- name: Find MSBuild
|
||||
shell: pwsh
|
||||
run: |
|
||||
$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
|
||||
$msbuild = & $vswhere -latest -products * -requires Microsoft.Component.MSBuild `
|
||||
-find MSBuild\**\Bin\MSBuild.exe | Select-Object -First 1
|
||||
if (-not $msbuild) { throw "MSBuild not found" }
|
||||
Write-Host "MSBuild: $msbuild"
|
||||
"MSBUILD=$msbuild" | Out-File -FilePath $env:GITHUB_ENV -Append
|
||||
|
||||
- name: Enable vcpkg MSBuild integration
|
||||
shell: pwsh
|
||||
run: vcpkg integrate install
|
||||
|
||||
# vcpkg errors if VCPKG_DEFAULT_BINARY_CACHE points at a missing dir, and
|
||||
# actions/cache does not create it on a miss.
|
||||
- name: Create the vcpkg binary cache directory
|
||||
shell: pwsh
|
||||
run: New-Item -ItemType Directory -Force -Path $env:VCPKG_DEFAULT_BINARY_CACHE | Out-Null
|
||||
|
||||
# x-gha is gone (vcpkg-tool #1662 dropped it after GitHub changed the cache
|
||||
# API), so cache the binary archives directly. Keyed on the manifest, which
|
||||
# carries the builtin-baseline, so a Dependabot bump busts it; restore-keys
|
||||
# still seeds the unchanged ports' archives, so only the bumped one rebuilds.
|
||||
- name: Cache vcpkg binary archives
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ github.workspace }}\vcpkg_cache
|
||||
key: vcpkg-${{ matrix.platform }}-${{ hashFiles('src/vcpkg.json') }}
|
||||
restore-keys: |
|
||||
vcpkg-${{ matrix.platform }}-
|
||||
|
||||
# The runner image's vcpkg checkout is pinned to some commit; our manifest's
|
||||
# builtin-baseline is usually newer, so `git show <baseline>:versions/...`
|
||||
# fails until that commit is local. Fetch exactly the pinned baseline (read
|
||||
# from the manifest, so Dependabot bumps need no workflow edit).
|
||||
- name: Fetch the pinned vcpkg baseline
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
$root = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { "C:\vcpkg" }
|
||||
$baseline = (Get-Content src/vcpkg.json -Raw | ConvertFrom-Json).'builtin-baseline'
|
||||
if (-not $baseline) { throw "no builtin-baseline in src/vcpkg.json" }
|
||||
git -C $root fetch --no-tags origin $baseline
|
||||
if ($LASTEXITCODE -ne 0) { throw "could not fetch vcpkg baseline $baseline" }
|
||||
|
||||
# httrack and webhttrack carry a ProjectReference to libhttrack, so building
|
||||
# them builds it first; proxytrack is standalone.
|
||||
- name: Build
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($proj in @("libhttrack", "httrack", "webhttrack", "proxytrack")) {
|
||||
& $env:MSBUILD "src\$proj.vcxproj" `
|
||||
/m `
|
||||
/p:Configuration=${{ matrix.configuration }} `
|
||||
/p:Platform=${{ matrix.platform }} `
|
||||
/p:VcpkgEnableManifest=true `
|
||||
/flp:LogFile=msbuild-$proj.log`;Verbosity=normal
|
||||
if ($LASTEXITCODE -ne 0) { throw "$proj failed" }
|
||||
}
|
||||
|
||||
# WinHTTrack links src\$(Platform)\$(Configuration)\libhttrack.lib, so the
|
||||
# import lib and the DLL both have to land there under that exact name.
|
||||
- name: Check the binaries landed where they are expected
|
||||
shell: pwsh
|
||||
run: |
|
||||
$dir = "src\${{ matrix.platform }}\${{ matrix.configuration }}"
|
||||
$want = @("libhttrack.dll", "libhttrack.lib", "httrack.exe",
|
||||
"webhttrack.exe", "proxytrack.exe")
|
||||
foreach ($f in $want) {
|
||||
if (-not (Test-Path "$dir\$f")) { throw "missing $dir\$f" }
|
||||
Write-Host "found $dir\$f"
|
||||
}
|
||||
|
||||
# A CRT split across the exe/DLL boundary links fine and crashes on the
|
||||
# first cross-heap free, so assert both sides import the *same* vcruntime.
|
||||
# Also assert the machine type: Test-Path alone would accept an x64 binary
|
||||
# sitting in the Win32 output dir.
|
||||
- name: Check the CRT is shared across the boundary
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
$dir = "src\${{ matrix.platform }}\${{ matrix.configuration }}"
|
||||
$dumpbin = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" `
|
||||
-latest -find VC\Tools\MSVC\**\bin\Host*\*\dumpbin.exe | Select-Object -First 1
|
||||
if (-not $dumpbin) { throw "dumpbin not found" }
|
||||
$wantMachine = if ("${{ matrix.platform }}" -eq "x64") { "x64" } else { "x86" }
|
||||
|
||||
function Get-Crt($path) {
|
||||
$out = & $dumpbin /dependents /headers $path
|
||||
if ($LASTEXITCODE -ne 0) { throw "dumpbin failed on $path" }
|
||||
,@($out | Select-String -Pattern '^\s*(vcruntime\d+d?\.dll)$' |
|
||||
ForEach-Object { $_.Matches[0].Groups[1].Value.ToLower() })
|
||||
}
|
||||
|
||||
# Negative control: cmd.exe links the old msvcrt, not vcruntime140. If
|
||||
# this came back non-empty the detector would match anything, and the
|
||||
# assertion below would pass on a mismatched CRT too.
|
||||
if ((Get-Crt "$env:SystemRoot\System32\cmd.exe").Count -ne 0) {
|
||||
throw "negative control failed: the vcruntime detector matches anything"
|
||||
}
|
||||
|
||||
$crts = @{}
|
||||
foreach ($f in @("libhttrack.dll", "httrack.exe")) {
|
||||
$crt = Get-Crt "$dir\$f"
|
||||
if ($crt.Count -eq 0) { throw "$f imports no vcruntime: static or mismatched CRT" }
|
||||
$crts[$f] = $crt
|
||||
|
||||
$hdr = & $dumpbin /headers "$dir\$f"
|
||||
if (-not ($hdr | Select-String -SimpleMatch "machine ($wantMachine)")) {
|
||||
throw "$f is not $wantMachine"
|
||||
}
|
||||
Write-Host "$f -> $($crt -join ',') ($wantMachine)"
|
||||
}
|
||||
|
||||
$a = $crts["libhttrack.dll"] | Sort-Object
|
||||
$b = $crts["httrack.exe"] | Sort-Object
|
||||
if (Compare-Object $a $b) {
|
||||
throw "CRT mismatch across the boundary: libhttrack.dll=$a httrack.exe=$b"
|
||||
}
|
||||
|
||||
# The engine ships ~90 tests and none had ever run on Windows: "make check"
|
||||
# is Linux/macOS only. These are the offline ones, driven from Git Bash
|
||||
# against the native httrack.exe. They subsume the self-tests this step
|
||||
# used to run inline (codecs, cache, fsize).
|
||||
# The *_local-* ones crawl the bundled Python server over loopback: the real
|
||||
# TLS handshake, cache and file writer, which nothing else on Windows covers.
|
||||
- name: Run the engine test suite (offline tests)
|
||||
shell: bash
|
||||
working-directory: tests
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
set -u
|
||||
bin="$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"
|
||||
export PATH="$bin:$PATH"
|
||||
command -v httrack >/dev/null || { echo "::error::no httrack.exe in $bin"; exit 1; }
|
||||
|
||||
# httrack.exe is native, so MSYS rewrites any argument shaped like a
|
||||
# POSIX path, and a URL path is shaped exactly like one: "/a/b.html"
|
||||
# reached the engine as "C:/Program Files/Git/a/b.html". Switch that
|
||||
# off, and hand the tests a TMPDIR that is already a Windows path.
|
||||
export MSYS_NO_PATHCONV=1
|
||||
export MSYS2_ARG_CONV_EXCL='*'
|
||||
TMPDIR="$(cygpath -m "$RUNNER_TEMP")"
|
||||
export TMPDIR
|
||||
|
||||
# Mirror what configure hands the suite. LC_ALL sets the codeset MSYS maps
|
||||
# a UTF-8 mirror name onto UTF-16 with, which the intl crawls "test -f".
|
||||
export HTTPS_SUPPORT=yes BROTLI_ENABLED=yes ZSTD_ENABLED=yes
|
||||
export LC_ALL=C.UTF-8
|
||||
|
||||
# A wedged crawl must not eat the job's timeout budget. timeout(1)'s
|
||||
# signals can't reap a native httrack.exe (MSYS signals don't reach it),
|
||||
# so a hang orphaned processes that starved the runner; run_with_timeout
|
||||
# TerminateProcess-es the whole tree. 600s is unchanged: it clears the
|
||||
# 540s a three-pass crawl may legitimately take under local-crawl.sh's
|
||||
# own watchdogs, against a slowest healthy test here of 39s.
|
||||
. ./testlib.sh
|
||||
per_test=600
|
||||
|
||||
# The whole suite must give up before the step timeout above. A cancelled
|
||||
# step keeps neither its log nor the artifacts the later if:always()
|
||||
# steps would upload, so an overrun that ends in a cancel tells us
|
||||
# nothing; failing on our own terms keeps both. Healthy runs take 8-9
|
||||
# min. The check sits between tests, so the step can still reach 25 min
|
||||
# plus one per-test budget, and that worst case stays inside the 45.
|
||||
suite_deadline=1500
|
||||
started=$SECONDS
|
||||
|
||||
# Survives into the artifact even if the tail of the step log does not.
|
||||
progress=suite-progress.log
|
||||
: >"$progress"
|
||||
|
||||
pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
|
||||
# Globbed, not enumerated: a new NNN_engine-*.test or NNN_local-*.test
|
||||
# is picked up automatically instead of silently getting zero coverage.
|
||||
for t in 00_runnable.test *_engine-*.test *_zlib-*.test \
|
||||
*_local-*.test *_watchdog*.test *_crawl_proxy_https.test \
|
||||
*_crawl-log-salvage.test; do
|
||||
elapsed=$((SECONDS - started))
|
||||
if [ "$elapsed" -ge "$suite_deadline" ]; then
|
||||
echo "::error::suite deadline: ${elapsed}s elapsed, stopping before $t"
|
||||
echo "DEADLINE before $t after ${elapsed}s" >>"$progress"
|
||||
# Per-test start times, so the slow ones are named rather than guessed.
|
||||
sed 's/^/ /' "$progress"
|
||||
deadline=1
|
||||
break
|
||||
fi
|
||||
echo "RUN $t at ${elapsed}s" >>"$progress"
|
||||
rc=0
|
||||
# Same guard "make check" uses on POSIX, so a wedge is diagnosed the
|
||||
# same way on every platform. It dumps before it kills, which a bare
|
||||
# run_with_timeout cannot: by the time that returns, the tree whose
|
||||
# stack we wanted is already gone.
|
||||
HTTRACK_TEST_TIMEOUT=$per_test bash ./test-timeout.sh "$t" >"$t.log" 2>&1 || rc=$?
|
||||
case "$rc" in
|
||||
0) pass=$((pass + 1)); echo "PASS $t" ;;
|
||||
77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
|
||||
124)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
# test-timeout.sh has already written the process list, the stacks
|
||||
# and the killed crawl's own logs into $t.log.
|
||||
echo "FAIL $t (timed out, tree killed)"
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
*)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (exit $rc)"
|
||||
# These assert with `test "$(...)" == "..." || exit 1`, which
|
||||
# says nothing at all on failure. Re-run traced, still bounded.
|
||||
run_with_timeout "$per_test" bash -x "$t" >>"$t.log" 2>&1 || true
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
esac
|
||||
echo "$rc $t" >>"$progress"
|
||||
# An orphaned native httrack.exe spins and starves the runner, which
|
||||
# is how this job dies with "lost communication" rather than a plain
|
||||
# timeout. Clear them between tests and name whoever leaked them.
|
||||
reap_leftover_processes "$t" | tee -a "$progress"
|
||||
done
|
||||
echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
|
||||
tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
# One name per line, so two branches each appending one don't collide on the
|
||||
# same line; compared as a sorted set below, so glob discovery order can't
|
||||
# cause a false mismatch either.
|
||||
# footer-overflow and purge-longpath skip on Windows (need a path past MAX_PATH);
|
||||
# crange pending #581;
|
||||
# webdav-default, webdav-mime and webdav-overflow need a reapable background listener, which MSYS cannot give them;
|
||||
# badmtime needs a filesystem that stores an mtime past gmtime's range;
|
||||
# single-file ends on a GUI half needing htsserver, which this job does not build;
|
||||
# update-304-leak needs a LeakSanitizer build, which MSVC has no equivalent of;
|
||||
# crash-symbolize and backtrace-empty need backtrace(), which Windows has no
|
||||
# equivalent of;
|
||||
# string-oom drives a helper binary that only the automake build produces.
|
||||
expected_skips="01_engine-footer-overflow.test
|
||||
100_local-purge-longpath.test
|
||||
114_local-update-304-leak.test
|
||||
120_local-proxytrack-webdav-default.test
|
||||
143_engine-backtrace-empty.test
|
||||
147_local-proxytrack-webdav-overflow.test
|
||||
152_engine-string-oom.test
|
||||
48_local-crange-memresume.test
|
||||
71_local-crange-repaircache.test
|
||||
79_local-proxytrack-webdav-mime.test
|
||||
80_engine-crash-symbolize.test
|
||||
88_local-proxytrack-badmtime.test
|
||||
94_local-single-file.test"
|
||||
# First, or the deadline reads as an unexplained shortfall in the gates below.
|
||||
[ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
# Word-split on whitespace (space-joined $skipped, newline-joined
|
||||
# expected_skips both work) and sort, so the compare is a set, not a string.
|
||||
got=$(printf '%s\n' $skipped | sort)
|
||||
want=$(printf '%s\n' $expected_skips | sort)
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo "::error::skip set changed from expected; - missing, + newly skipped"
|
||||
diff -u <(echo "$want") <(echo "$got") | tail -n +3 | sed 's/^/ /'
|
||||
exit 1
|
||||
fi
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
- name: Upload the test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: engine-tests-${{ matrix.platform }}-${{ matrix.configuration }}
|
||||
path: tests/*.log
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload MSBuild logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: msbuild-${{ matrix.platform }}-${{ matrix.configuration }}
|
||||
path: msbuild-*.log
|
||||
if-no-files-found: ignore
|
||||
11
.gitignore
vendored
11
.gitignore
vendored
@@ -24,6 +24,8 @@ Makefile.in
|
||||
/config.log
|
||||
/config.status
|
||||
/stamp-h1
|
||||
# src/webhttrack.in is the source; an in-tree build generates this one (#887).
|
||||
/src/webhttrack
|
||||
Makefile
|
||||
.deps/
|
||||
.libs/
|
||||
@@ -34,8 +36,15 @@ Makefile
|
||||
*.so.*
|
||||
*.a
|
||||
|
||||
# make dist output.
|
||||
# make dist output; dist/ holds httrack-gh-release staging artifacts.
|
||||
/httrack-*.tar.gz
|
||||
/dist/
|
||||
|
||||
# Editor / autotools backup files.
|
||||
*~
|
||||
|
||||
# Python bytecode (tests/local-server.py).
|
||||
__pycache__/
|
||||
|
||||
# Per-checkout Claude Code rules (symlink into a local sandbox).
|
||||
/CLAUDE.local.md
|
||||
|
||||
88
AGENTS.md
88
AGENTS.md
@@ -6,8 +6,38 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
## Build & test
|
||||
- Fresh clone first: `git submodule update --init src/coucal`
|
||||
- `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
|
||||
automake, libtool), then `bash configure && make && make check`. Or run
|
||||
`sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
automake, libtool), then `bash configure && make -j"$(nproc)" && make check
|
||||
-j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
|
||||
automake's parallel harness and each crawl test binds its own ephemeral-port
|
||||
server, so `-j` never contends and a multi-minute serial run drops to
|
||||
seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
|
||||
automatically; only a test slower than the current longest raises the floor.
|
||||
On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
|
||||
spend much of their wall time asleep (server trickles, httrack self-pacing),
|
||||
so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
|
||||
platform, macOS included: the test server raises its listen backlog
|
||||
(`request_queue_size`) so macOS/BSD don't drop connections under a parallel
|
||||
`-c16` bigcrawl the way Python's default backlog of 5 did.
|
||||
Or run `sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
- A `tests/NN_*.test` runs only if listed in `tests/tests-list.mk`; an
|
||||
unregistered file is silently skipped.
|
||||
- `make check` prepends the build's `src/` to `PATH`, but a hand-run `.test` does
|
||||
not — an installed `/usr/bin/httrack` then shadows your build. Run via `make
|
||||
check`, or `PATH="<bld>/src:$PATH"` for a manual run.
|
||||
- Give new `.test` scripts `set -e`: the older ones predate the rule, so several
|
||||
`local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
|
||||
- Run teardown with errexit off: `trap 'set +e; cleanup' EXIT`. Under `set -e` a
|
||||
failing cleanup command becomes the test's exit status (#773). Keep the other
|
||||
signals on their own `trap` line, or errexit stays off for the rest of the run.
|
||||
The guard also resets `$?`, so save it first if teardown reads it.
|
||||
- Never pipe into `grep -q`: it exits on the first match, so whatever the
|
||||
producer had left to write takes SIGPIPE, and under `pipefail` that becomes
|
||||
the pipeline's status. `cmd | grep -q M && fail` then never fires and a probe
|
||||
that proved nothing reads as "marker absent"; `cmd | grep -q M || fail` fails
|
||||
a test whose marker was present. bash issues one `write()` per line, so any
|
||||
match that is not on the last line is exposed. Capture the reply, assert the
|
||||
status line it must carry (an empty, truncated or redirected one is
|
||||
marker-free too), then match with a here-string: `grep -q M <<<"$reply"`.
|
||||
|
||||
## Hard invariants
|
||||
- **Generated autotools files are NOT in git.** `configure`, every
|
||||
@@ -20,21 +50,49 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
- **Format only changed lines** with `git clang-format` (clang-format 19). Never
|
||||
reformat untouched code: the engine was formatted by an old tool and won't
|
||||
round-trip.
|
||||
- **Byte-safe edits.** Files with raw high bytes are ISO-8859-1 (French
|
||||
comments). Edit them byte-wise (`perl -0pi`, `sed`), not through a tool that
|
||||
re-encodes to UTF-8 and corrupts them.
|
||||
- **Byte-safe edits.** A few tracked files carry raw ISO-8859-1 high bytes
|
||||
(French comments): `src/htsconcat.c`, `lang/*.txt`, `html/contact.html`, and
|
||||
the `fuzz/corpus/*` vectors. Edit those byte-wise (`perl -0pi`, `sed`), not
|
||||
through a tool that re-encodes to UTF-8 and corrupts them. The rest of the tree
|
||||
is UTF-8 and safe to edit normally.
|
||||
|
||||
## Security (HTTrack parses hostile input off the network)
|
||||
- Bounds-check every copy. Overflow-safe form: put the untrusted value alone,
|
||||
`untrusted < limit - controlled` — never `controlled + untrusted < limit`,
|
||||
which can wrap and pass.
|
||||
- **Abort or clip is a decision, not a default.** The `*_safe_` helpers
|
||||
(`strcpybuff`, `strlcpybuff`, `strcatbuff`) **abort** on overflow. Right for
|
||||
our own data, wrong for anything read back from a cache, a header or the
|
||||
wire, where it trades a memory smash for a crash on malformed input. Clip
|
||||
with `dst[0] = '\0'; strlncatbuff(dst, src, size, size - 1)`.
|
||||
- **A warning class is not the unsafe set.** `-Wformat-truncation` fires only on
|
||||
a *bounded* `snprintf` whose return is discarded, so an unbounded `sprintf`
|
||||
into the same buffer never appears on it. Before scoping a hardening pass off
|
||||
compiler output, grep the unguarded forms yourself (`\bsprintf\s*\(`,
|
||||
`\bstrcpy\s*\(`, `\bstrcat\s*\(`).
|
||||
|
||||
## C conventions
|
||||
- **Use the `*t` allocator wrappers, never raw libc** (`htssafe.h`):
|
||||
`malloct`/`calloct`/`realloct`/`freet`/`strdupt`, in test and selftest code
|
||||
too. `freet` NULLs its (lvalue) argument and tolerates NULL; `calloct(n, sz)`
|
||||
keeps calloc's arg order. Only exception: storing or calling a libc symbol
|
||||
itself (e.g. a resolver-backend function pointer).
|
||||
- **Exported API is `HTSEXT_API`.** Everything else is hidden by
|
||||
`-fvisibility=hidden` and free to change (check with `nm -D --defined-only
|
||||
libhttrack.so`). Touching an installed-header struct (see `DevIncludes_DATA` in
|
||||
`src/Makefile.am`) or an exported signature is an ABI break — flag and discuss,
|
||||
bump the soname, and prefer keeping the old entry point beside a new one.
|
||||
- **Windows ABI is free to break, POSIX is not.** The Windows DLL ships next to
|
||||
the exe with no soname contract, so a `_WIN32`-only ABI change needs no
|
||||
deprecation dance; POSIX/ELF keeps the flag-discuss-bump rules.
|
||||
|
||||
## Code & prose
|
||||
- Be terse. Comment the why, in English; translate French comments you touch.
|
||||
- Strip AI tells from prose (em-dash overuse, rule-of-three, filler, vague
|
||||
attributions). Ref: Wikipedia "Signs of AI writing". Claude Code: `/humanizer`.
|
||||
- Behavior change → add a test. Fast path: a hidden `httrack -#N` debug
|
||||
subcommand (`htscoremain.c`) driven by a `tests/NN_*.test`, over a slow crawl.
|
||||
- Behavior change → add a test. Fast path: a hidden `httrack -#test=NAME` engine
|
||||
self-test (registry in `htsselftest.c`; `-#test` lists them) driven by a
|
||||
`tests/NN_*.test`, over a slow crawl.
|
||||
|
||||
## Review your change adversarially (strongly suggested)
|
||||
Before pushing, and when reviewing others, don't skim for bugs:
|
||||
@@ -48,6 +106,17 @@ Before pushing, and when reviewing others, don't skim for bugs:
|
||||
layout/ABI, cache/wire format, or a security path? A static or unit check
|
||||
isn't enough; exercise the wrong behavior at runtime. Claude Code:
|
||||
`/review-recipe`.
|
||||
- **Poison a canary, never compare it against zero.** Checking that a
|
||||
neighbouring field is still `'\0'` cannot see the stray NUL an off-by-one
|
||||
terminator writes — the exact bug the canary is there for. Fill it with a
|
||||
non-zero byte, and prove it by killing both the stray-`'X'` and the
|
||||
stray-NUL mutant. Neither ASan nor `_FORTIFY_SOURCE` sees an overflow that
|
||||
lands inside the same struct.
|
||||
- **Overshoot every destination, not one.** A bounds test that oversizes a
|
||||
single field cannot tell a per-field bound from a one-size-fits-all one, nor
|
||||
from a fix that bounds that field and leaves its neighbours raw. Exercise
|
||||
each destination the path touches, spanning at least two capacities, and
|
||||
check what the code actually emits before writing the expected values.
|
||||
|
||||
## Commits
|
||||
- **Sign-off is mandatory.** Every commit carries a `Signed-off-by` trailer:
|
||||
@@ -55,8 +124,9 @@ Before pushing, and when reviewing others, don't skim for bugs:
|
||||
- **Co-Authored-By is mandatory for AI-assisted commits.** Carry a
|
||||
`Co-Authored-By:` trailer naming the assistant. Attribute there, never in a
|
||||
PR-body footer.
|
||||
- PRs land as a merge commit; every commit on the branch goes onto master, so
|
||||
keep each commit message clean and meaningful.
|
||||
- PRs are squash-merged: one commit per PR lands on master, built from the PR
|
||||
title and description, so those are what the history keeps. The branch's
|
||||
intermediate commits are not preserved.
|
||||
|
||||
## PR descriptions
|
||||
- Plain concise prose; lead with what changed and why. No What/Why/How template.
|
||||
|
||||
@@ -6,8 +6,8 @@ with an AI assistant? The operational checklist is [AGENTS.md](AGENTS.md).
|
||||
## Pull requests
|
||||
|
||||
- One change per PR. Small diffs merge fast.
|
||||
- PRs land as a merge commit, so the branch's commits go onto master as-is: keep
|
||||
each commit message clean and explain *why*.
|
||||
- PRs are squash-merged: one commit per PR goes onto master, built from the PR
|
||||
title and description, so those are the history. Explain *why* there.
|
||||
- Be terse in the PR title and description: name the problem, not the fix, don't
|
||||
restate the diff, and calibrate length to the change.
|
||||
- Add or update tests for engine changes (`tests/`), and keep CI green.
|
||||
@@ -39,6 +39,10 @@ Welcome, and nothing to disclose. Two rules:
|
||||
|
||||
The sign-off covers AI-assisted code too.
|
||||
|
||||
## Translations
|
||||
|
||||
Interface strings live in [`lang/`](lang/). See [lang/README.md](lang/README.md) for the file format and how to add or update a language.
|
||||
|
||||
## Bugs
|
||||
|
||||
Open an issue with the version, OS, command used, and expected vs actual result.
|
||||
|
||||
14
Makefile.am
14
Makefile.am
@@ -1,18 +1,26 @@
|
||||
|
||||
SUBDIRS = src man m4 libtest templates lang html tests
|
||||
SUBDIRS = src man m4 libtest templates lang html tests fuzz
|
||||
|
||||
ACLOCAL_AMFLAGS = -I m4
|
||||
|
||||
EXTRA_DIST = INSTALL.Linux \
|
||||
gpl-fr.txt license.txt greetings.txt history.txt \
|
||||
httrack-doc.html lang.def README.md tools/mkdeb.sh \
|
||||
tools/macos-app.sh tools/Info.plist.in \
|
||||
bootstrap build.sh
|
||||
|
||||
# Build the signed Debian packages from a clean source export. Pass the signing
|
||||
# key and other options through DEB_FLAGS, e.g.:
|
||||
# make deb DEB_FLAGS="--key BB71C7E6CB1AD8FAF53FE42A60C3AA7180598EFB"
|
||||
# See tools/mkdeb.sh --help for all options.
|
||||
# See tools/mkdeb.sh --help for all options. Not $(SHELL): mkdeb.sh is bash, and dash chokes (#891).
|
||||
DEB_FLAGS =
|
||||
deb:
|
||||
$(SHELL) $(top_srcdir)/tools/mkdeb.sh $(DEB_FLAGS)
|
||||
$(BASH_SHELL) $(top_srcdir)/tools/mkdeb.sh $(DEB_FLAGS)
|
||||
.PHONY: deb
|
||||
|
||||
# Assemble the macOS application bundle from an installed prefix, e.g.
|
||||
# make macos-app APP_FLAGS="--prefix /tmp/inst --out /tmp"
|
||||
APP_FLAGS =
|
||||
macos-app:
|
||||
$(SHELL) $(top_srcdir)/tools/macos-app.sh --plist tools/Info.plist $(APP_FLAGS)
|
||||
.PHONY: macos-app
|
||||
|
||||
1
README
1
README
@@ -40,7 +40,6 @@ These are the principals limits of HTTrack for that moment. Note that we did not
|
||||
that would have solved them.
|
||||
|
||||
- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
|
||||
- Some java classes may not find some files on them (class included)
|
||||
- Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*
|
||||
|
||||
|
||||
|
||||
125
configure.ac
125
configure.ac
@@ -1,6 +1,6 @@
|
||||
AC_PREREQ([2.71])
|
||||
|
||||
AC_INIT([httrack], [3.49.8], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_INIT([httrack], [3.49.15], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_COPYRIGHT([
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2015 Xavier Roche and other contributors
|
||||
@@ -29,9 +29,12 @@ AC_CONFIG_SRCDIR(src/httrack.c)
|
||||
AC_CONFIG_MACRO_DIR([m4])
|
||||
AC_CONFIG_HEADERS(config.h)
|
||||
AM_INIT_AUTOMAKE([subdir-objects])
|
||||
# 3:0:0: htsblk layout changed (contenttype/charset/contentencoding widened to
|
||||
# 128), an incompatible ABI break, so bump current and reset revision/age.
|
||||
VERSION_INFO="3:0:0"
|
||||
# 3:7:0: htsblk gained a tail field and lien_back embeds it by value, so
|
||||
# lien_back.is_update and everything after it shift +8 (httrackp's own tail growth
|
||||
# moves nothing). Soname stays .so.3: HTTrackQt is the only consumer of the installed
|
||||
# headers, so a libhttrack4 rename isn't worth it.
|
||||
# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
|
||||
VERSION_INFO="3:7:0"
|
||||
AM_MAINTAINER_MODE
|
||||
AC_USE_SYSTEM_EXTENSIONS
|
||||
|
||||
@@ -45,13 +48,18 @@ m4_warn([obsolete],
|
||||
# script's behavior did not change. They are probably safe to remove.
|
||||
AC_CHECK_INCLUDES_DEFAULT
|
||||
AC_PROG_EGREP
|
||||
# $(SED) substitutes $(datadir) into src/webhttrack
|
||||
AC_PROG_SED
|
||||
|
||||
LT_INIT
|
||||
AC_PROG_LN_S
|
||||
LT_INIT
|
||||
|
||||
# bash, used to run the test scripts (see tests/Makefile.am TEST_LOG_COMPILER)
|
||||
AC_PATH_PROGS([BASH], [bash], [/bin/bash])
|
||||
# A real bash, for "make deb" and the test harness. Not searched into BASH: bash presets
|
||||
# that to its own path, and macOS /bin/sh is a bash, so the macro never searches (#895).
|
||||
# BASH_SHELL isn't preset the way BASH is, so AC_ARG_VAR needs no guard.
|
||||
AC_ARG_VAR([BASH_SHELL], [path to a real (non-POSIX-mode) bash])
|
||||
AC_PATH_PROGS([BASH_SHELL], [bash], [/bin/bash])
|
||||
|
||||
# Export LD_LIBRARY_PATH name or equivalent.
|
||||
AC_SUBST(SHLIBPATH_VAR,$shlibpath_var)
|
||||
@@ -63,10 +71,11 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
|
||||
AC_SUBST(VERSION_INFO)
|
||||
|
||||
### Default CFLAGS
|
||||
# No -Wdeclaration-after-statement: nothing sets -std=, so this builds as gnu17.
|
||||
DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
|
||||
-Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
|
||||
-Wstrict-prototypes -Wmissing-prototypes \
|
||||
-Wmissing-declarations -Wdeclaration-after-statement \
|
||||
-Wmissing-declarations \
|
||||
-Wpointer-arith -Wsequence-point -Wnested-externs \
|
||||
-D_REENTRANT"
|
||||
AC_SUBST(DEFAULT_CFLAGS)
|
||||
@@ -74,33 +83,47 @@ DEFAULT_LDFLAGS=""
|
||||
AC_SUBST(DEFAULT_LDFLAGS)
|
||||
|
||||
### Additional flags (if supported)
|
||||
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"])
|
||||
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"])
|
||||
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"])
|
||||
# -Werror on probes: exit-status-only checks let clang's warn-on-unknown-flag through.
|
||||
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"], [], [-Werror])
|
||||
# -Wformat-nonliteral needs -Wformat in the probe or gcc rejects it as ignored.
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"], [], [-Werror -Wformat])
|
||||
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"], [], [-Werror])
|
||||
# Make htssafe.h's pointer-dest 'warning' attribute a hard error in our build
|
||||
# (migration is at zero; a new char* dest is a regression). gcc/clang each take
|
||||
# only their own spelling; downstream keeps the plain warning, not a build break.
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
|
||||
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
|
||||
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
|
||||
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"], [], [-Werror])], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"], [], [-Werror])
|
||||
# backtrace() unwinds through these; armhf, unlike amd64/arm64, defaults them off
|
||||
# and printed frameless crash reports.
|
||||
AX_CHECK_COMPILE_FLAG([-fasynchronous-unwind-tables], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fasynchronous-unwind-tables"], [], [-Werror])
|
||||
# No --discard-all: it drops the local symbols naming every static function, so
|
||||
# a trace misattributes them to the nearest surviving global. Costs 0.6% size.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--no-undefined"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
|
||||
|
||||
# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
|
||||
# sanitizer builds, whose interceptors want the unfortified calls.
|
||||
case "$CFLAGS" in
|
||||
*-fsanitize=*) ;;
|
||||
*) AX_ADD_FORTIFY_SOURCE ;;
|
||||
esac
|
||||
|
||||
# Force libc back into DT_NEEDED for libraries that reach it only through
|
||||
# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
|
||||
# linker; Apple ld rejects these flags and links libSystem unconditionally.
|
||||
# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
|
||||
# ld rejects these flags and links libSystem unconditionally.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
|
||||
[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
|
||||
AC_SUBST([LIBC_FORCE_LINK])
|
||||
@@ -108,13 +131,13 @@ AC_SUBST([LIBC_FORCE_LINK])
|
||||
### PIE
|
||||
CFLAGS_PIE=""
|
||||
LDFLAGS_PIE=""
|
||||
AX_CHECK_COMPILE_FLAG([-fpie -pie], [CFLAGS_PIE="-fpie -pie"])
|
||||
AX_CHECK_COMPILE_FLAG([-fpie], [CFLAGS_PIE="-fpie"], [], [-Werror])
|
||||
AX_CHECK_LINK_FLAG([-pie], [LDFLAGS_PIE="-pie"])
|
||||
AC_SUBST([CFLAGS_PIE])
|
||||
AC_SUBST([LDFLAGS_PIE])
|
||||
|
||||
## Export all symbols for backtraces
|
||||
AX_CHECK_COMPILE_FLAG([-rdynamic], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -rdynamic"])
|
||||
# Ties a crash trace from a stripped build back to its separate debug symbols.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--build-id], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--build-id"])
|
||||
|
||||
### Check for -fvisibility=hidden support
|
||||
gl_VISIBILITY
|
||||
@@ -158,11 +181,15 @@ AC_CHECK_TYPE(sa_family_t, [], [AC_DEFINE([sa_family_t], [uint16_t], [sa_family_
|
||||
AX_CHECK_ALIGNED_ACCESS_REQUIRED
|
||||
|
||||
# check for various headers
|
||||
AC_CHECK_HEADERS([execinfo.h])
|
||||
AC_CHECK_HEADERS([execinfo.h sys/ioctl.h])
|
||||
|
||||
### zlib
|
||||
### zlib (mandatory)
|
||||
CHECK_ZLIB()
|
||||
|
||||
### brotli and zstd content codings (optional)
|
||||
CHECK_BROTLI()
|
||||
CHECK_ZSTD()
|
||||
|
||||
### OpenSSL is explicitly enabled/disabled ?
|
||||
AC_MSG_CHECKING(whether to enable https support)
|
||||
AC_ARG_ENABLE([https],
|
||||
@@ -214,9 +241,12 @@ AC_SUBST(OPENSSL_LIBS)
|
||||
fi
|
||||
|
||||
### Support IPv6
|
||||
V6_SUPPORT=no
|
||||
AC_CHECK_LIB(c, getaddrinfo, [V6_FLAG="-DINET6"
|
||||
V6_SUPPORT=yes
|
||||
AC_DEFINE(HTS_INET6, 1, [Check for IPv6])], AC_MSG_WARN([*** IPv6 not found IPv6 compatibility disabled]))
|
||||
AC_SUBST(V6_FLAG)
|
||||
AC_SUBST(V6_SUPPORT)
|
||||
|
||||
### Check for LFS
|
||||
AC_CHECK_LIB(c, fopen64, [LFS_FLAG="-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE"
|
||||
@@ -295,6 +325,37 @@ AC_ARG_ENABLE([online-unit-tests],
|
||||
])
|
||||
AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
|
||||
|
||||
## libFuzzer harnesses (fuzz/); requires clang
|
||||
AC_MSG_CHECKING(whether to build fuzzers)
|
||||
AC_ARG_ENABLE([fuzzers],
|
||||
[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
|
||||
[
|
||||
case "${enableval}" in
|
||||
no|yes)
|
||||
fuzzers=$enableval
|
||||
AC_MSG_RESULT($enableval)
|
||||
;;
|
||||
*)
|
||||
AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
|
||||
;;
|
||||
esac
|
||||
],
|
||||
[
|
||||
fuzzers=no
|
||||
AC_MSG_RESULT(no)
|
||||
])
|
||||
if test x"$fuzzers" = x"yes"; then
|
||||
# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
|
||||
AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
|
||||
[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
|
||||
[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
|
||||
# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
|
||||
if test x"$enable_shared" != x"no"; then
|
||||
AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
|
||||
fi
|
||||
fi
|
||||
AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
|
||||
|
||||
# Final output
|
||||
AC_CONFIG_FILES([
|
||||
Makefile
|
||||
@@ -306,5 +367,7 @@ lang/Makefile
|
||||
html/Makefile
|
||||
libtest/Makefile
|
||||
tests/Makefile
|
||||
fuzz/Makefile
|
||||
tools/Info.plist
|
||||
])
|
||||
AC_OUTPUT
|
||||
|
||||
101
debian/changelog
vendored
101
debian/changelog
vendored
@@ -1,3 +1,104 @@
|
||||
httrack (3.49.15-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: single-file mirrors with inlined assets, sitemap
|
||||
ingestion, a change report between crawls, and numerous --update, FTP and
|
||||
parser fixes; full list in history.txt.
|
||||
* httrack-doc grows by roughly 1 MB: the offline guide now ships screenshots
|
||||
of the WinHTTrack, WebHTTrack and Android interfaces.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Thu, 30 Jul 2026 10:40:08 +0200
|
||||
|
||||
httrack (3.49.14-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: WARC/WACZ archive output, Windows long-path support,
|
||||
named -%F footer fields, and encoding fixes for non-ASCII project paths;
|
||||
full list in history.txt.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Fri, 24 Jul 2026 08:01:43 +0200
|
||||
|
||||
httrack (3.49.13-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: SOCKS5 and CONNECT proxy support, brotli and zstd
|
||||
content decoding, --update data-loss fixes, and hardening of several
|
||||
network-facing parsers; full list in history.txt.
|
||||
* Add libbrotli-dev and libzstd-dev to Build-Depends for the new content
|
||||
codings.
|
||||
* libhttrack3 and libhttrack-dev no longer ship libhtsjava.so.3*: the
|
||||
obsolete Java-applet parser was removed upstream. It was a dlopen plugin
|
||||
with no reverse dependencies, so this is a plain file drop on upgrade;
|
||||
libhttrack.so.3 itself is unchanged and needs no rename.
|
||||
* Drop the htsjava attribution from debian/copyright.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Fri, 17 Jul 2026 10:49:52 +0200
|
||||
|
||||
httrack (3.49.12-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: security and crawl-correctness fixes (remote stack
|
||||
overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
|
||||
world-readable cookies.txt, filter-pattern denial of service) plus a new
|
||||
--why filter diagnostic; full list in history.txt.
|
||||
* Build with _FORTIFY_SOURCE and -fstack-protector-strong.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Fri, 10 Jul 2026 21:11:09 +0200
|
||||
|
||||
httrack (3.49.11-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: crawl correctness and security fixes (network-facing
|
||||
buffer overflows, file-type detection, redirect handling) and modernized
|
||||
web defaults; full list in history.txt.
|
||||
* Add DEP-12 upstream metadata (#466).
|
||||
* Bump debhelper compat to 14 (#466).
|
||||
* Drop the redundant Priority field and update the NMU lintian override to
|
||||
the current tag names (#466).
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 05 Jul 2026 00:03:18 +0200
|
||||
|
||||
httrack (3.49.10-2) unstable; urgency=medium
|
||||
|
||||
* Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
|
||||
absent (the local-server tests need python3, which the buildds lack). Add
|
||||
patches/skip-local-pause-test-without-python3.patch to guard the test on
|
||||
python3 up front, like its siblings, so it skips cleanly.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 28 Jun 2026 20:18:46 +0200
|
||||
|
||||
httrack (3.49.10-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: new download-pacing and URL-handling options plus a
|
||||
batch of crawl and robustness fixes (full list in history.txt).
|
||||
* Rewrite debian/copyright in machine-readable DEP-5 format, crediting the
|
||||
bundled minizip, md5 and coucal sources (#415).
|
||||
* Lead the webhttrack browser dependency with chromium so httrack is not
|
||||
dragged into the firefox-esr autoremoval cascade (#436).
|
||||
* Override the embedded-library lint for the bundled minizip (#419).
|
||||
* Bump Standards-Version to 4.7.4 (no changes required).
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 28 Jun 2026 14:01:53 +0200
|
||||
|
||||
httrack (3.49.9-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: Content-Type and file-type detection fixes (trust a
|
||||
declared Content-Type over a binary URL extension, honor --assume under the
|
||||
delayed type check, keep a known extension against a bogus or empty
|
||||
Content-Type, and avoid an uninitialised read on an empty Content-Type), and
|
||||
restored C++ source-compatibility of the installed headers so reverse
|
||||
dependencies (httraqt) build again.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 21 Jun 2026 17:59:38 +0200
|
||||
|
||||
httrack (3.49.8-2) unstable; urgency=medium
|
||||
|
||||
* Rename libhttrack2 to libhttrack3 to follow the SONAME, which the 3.49.8
|
||||
ABI bump moved to libhttrack.so.3 (package-name-doesnt-match-sonames). In
|
||||
3.49.8-1 the libhttrack2.files glob still matched .so.2, so the runtime
|
||||
libraries fell through into the httrack package and libhttrack2 shipped no
|
||||
library. The new .files uses a .so.3* wildcard so a future SONAME bump no
|
||||
longer silently misplaces the libraries. New binary package, via NEW.
|
||||
* Drop the stale debian/libhttrack-swf1.files: the swf module is no longer
|
||||
built and no libhttrack-swf1 package exists.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sat, 20 Jun 2026 14:42:13 +0200
|
||||
|
||||
httrack (3.49.8-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: HTTPS-proxy CONNECT tunnelling and wider srcset
|
||||
|
||||
13
debian/control
vendored
13
debian/control
vendored
@@ -1,9 +1,8 @@
|
||||
Source: httrack
|
||||
Section: web
|
||||
Priority: optional
|
||||
Maintainer: Xavier Roche <roche@httrack.com>
|
||||
Standards-Version: 4.7.0
|
||||
Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
|
||||
Standards-Version: 4.7.4
|
||||
Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev, libbrotli-dev, libzstd-dev
|
||||
Rules-Requires-Root: no
|
||||
Homepage: http://www.httrack.com
|
||||
Vcs-Git: https://github.com/xroche/httrack.git
|
||||
@@ -30,7 +29,7 @@ Description: Copy websites to your computer (Offline browser)
|
||||
Package: webhttrack
|
||||
Architecture: any
|
||||
Multi-Arch: foreign
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, firefox-esr | chromium | www-browser
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, chromium | firefox-esr | www-browser
|
||||
Replaces: webhttrack-common (<< 3.43.9-2)
|
||||
Breaks: webhttrack-common (<< 3.43.9-2)
|
||||
Suggests: httrack, httrack-doc
|
||||
@@ -58,13 +57,13 @@ Description: webhttrack common files
|
||||
This package is the common files of webhttrack, website copier and
|
||||
mirroring utility
|
||||
|
||||
Package: libhttrack2
|
||||
Package: libhttrack3
|
||||
Architecture: any
|
||||
Multi-Arch: same
|
||||
Section: libs
|
||||
Replaces: libhttrack1
|
||||
Conflicts: libhttrack1
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}
|
||||
Replaces: libhttrack2, httrack (<< 3.49.8-2~)
|
||||
Breaks: libhttrack2, httrack (<< 3.49.8-2~)
|
||||
Description: Httrack website copier library
|
||||
This package is the library part of httrack, website copier and mirroring
|
||||
utility
|
||||
|
||||
117
debian/copyright
vendored
117
debian/copyright
vendored
@@ -1,21 +1,108 @@
|
||||
This package was debianized by Xavier Roche <roche@httrack.com> on
|
||||
Fri, 27 Sep 2002 16:42:26 +0200
|
||||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Name: httrack
|
||||
Upstream-Contact: Xavier Roche <roche@httrack.com>
|
||||
Source: https://www.httrack.com/
|
||||
|
||||
The current Debian maintainer is Xavier Roche <xavier@debian.org>
|
||||
Files: *
|
||||
Copyright: 1998-2026 Xavier Roche and other contributors
|
||||
License: GPL-3+
|
||||
Comment:
|
||||
htsbasenet.h links against the system OpenSSL library (originally by Eric
|
||||
Young); no OpenSSL/SSLeay code is bundled here.
|
||||
|
||||
Upstream author: Xavier Roche <roche@httrack.com>
|
||||
Files: src/minizip/*
|
||||
Copyright: 1998-2010 Gilles Vollant
|
||||
2007-2008 Even Rouault
|
||||
2009-2010 Mathias Svensson
|
||||
1990-2000 Info-ZIP
|
||||
License: Zlib
|
||||
Comment:
|
||||
The decryption code in src/minizip/crypt.h and src/minizip/unzip.c derives
|
||||
from the Info-ZIP distribution, distributed under the same terms.
|
||||
|
||||
Copyright: 1998-2014 Xavier Roche and other contributors
|
||||
Files: src/md5.c
|
||||
Copyright: 1993 Colin Plumb
|
||||
License: public-domain-md5
|
||||
This code implements the MD5 message-digest algorithm, due to Ron Rivest.
|
||||
It was written by Colin Plumb in 1993, no copyright is claimed. This code
|
||||
is in the public domain; do with it what you wish.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
Files: src/coucal/*
|
||||
Copyright: 2013-2014 Xavier Roche
|
||||
License: BSD-3-clause
|
||||
|
||||
On Debian systems, the complete text of the GNU General Public
|
||||
License version 3 can be found in /usr/share/common-licenses/GPL-3 file.
|
||||
Files: src/coucal/murmurhash3.h*
|
||||
Copyright: Austin Appleby
|
||||
License: public-domain-murmurhash3
|
||||
MurmurHash3 was written by Austin Appleby, and is placed in the public
|
||||
domain. The author hereby disclaims copyright to this source code.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
Files: html/server/div/com.httrack.WebHTTrack.metainfo.xml
|
||||
Copyright: 1998-2026 Xavier Roche and other contributors
|
||||
License: FSFAP
|
||||
Copying and distribution of this file, with or without modification, are
|
||||
permitted in any medium without royalty provided the copyright notice and
|
||||
this notice are preserved. This file is offered as-is, without any warranty.
|
||||
|
||||
Files: debian/*
|
||||
Copyright: 2002-2026 Xavier Roche <xavier@debian.org>
|
||||
License: GPL-3+
|
||||
|
||||
License: GPL-3+
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
.
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
.
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
.
|
||||
On Debian systems, the complete text of the GNU General Public License
|
||||
version 3 can be found in /usr/share/common-licenses/GPL-3.
|
||||
|
||||
License: Zlib
|
||||
This software is provided 'as-is', without any express or implied warranty.
|
||||
In no event will the authors be held liable for any damages arising from the
|
||||
use of this software.
|
||||
.
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
.
|
||||
1. The origin of this software must not be misrepresented; you must not claim
|
||||
that you wrote the original software. If you use this software in a product,
|
||||
an acknowledgment in the product documentation would be appreciated but is
|
||||
not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be
|
||||
misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
|
||||
License: BSD-3-clause
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
.
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
3. Neither the name of the copyright holder nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
3
debian/httrack-doc.lintian-overrides
vendored
3
debian/httrack-doc.lintian-overrides
vendored
@@ -4,3 +4,6 @@
|
||||
# so the path lives in the display pointer, not the override -- match with '*'.
|
||||
httrack-doc: extra-license-file *
|
||||
httrack-doc: package-contains-documentation-outside-usr-share-doc *
|
||||
# search.sh is a sample CGI shipped alongside the HTML manual, not meant to be
|
||||
# run from the package tree; it stays non-executable by design.
|
||||
httrack-doc: script-not-executable *
|
||||
|
||||
1
debian/libhttrack-dev.files
vendored
1
debian/libhttrack-dev.files
vendored
@@ -1,5 +1,4 @@
|
||||
usr/include/httrack
|
||||
usr/lib/*/libhttrack.{so}
|
||||
usr/lib/*/libhtsjava.{so}
|
||||
usr/share/httrack/libtest
|
||||
usr/lib/*/httrack/libtest
|
||||
|
||||
2
debian/libhttrack-swf1.files
vendored
2
debian/libhttrack-swf1.files
vendored
@@ -1,2 +0,0 @@
|
||||
usr/lib/*/libhtsswf.so.1.0.0
|
||||
usr/lib/*/libhtsswf.so.1
|
||||
5
debian/libhttrack2.files
vendored
5
debian/libhttrack2.files
vendored
@@ -1,5 +0,0 @@
|
||||
usr/lib/*/libhttrack.so.2.0.49
|
||||
usr/lib/*/libhttrack.so.2
|
||||
usr/lib/*/libhtsjava.so.2.0.49
|
||||
usr/lib/*/libhtsjava.so.2
|
||||
usr/share/httrack/templates
|
||||
3
debian/libhttrack2.lintian-overrides
vendored
3
debian/libhttrack2.lintian-overrides
vendored
@@ -1,3 +0,0 @@
|
||||
# The shared libraries ship without a versioned symbols control file (ABI is
|
||||
# tracked via the SONAME and a strict =version dependency, see debian/rules).
|
||||
libhttrack2: no-symbols-control-file usr/lib/*
|
||||
2
debian/libhttrack3.files
vendored
Normal file
2
debian/libhttrack3.files
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
usr/lib/*/libhttrack.so.3*
|
||||
usr/share/httrack/templates
|
||||
8
debian/libhttrack3.lintian-overrides
vendored
Normal file
8
debian/libhttrack3.lintian-overrides
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
# The shared libraries ship without a versioned symbols control file (ABI is
|
||||
# tracked via the SONAME plus a >= upstream-version dependency, see debian/rules).
|
||||
libhttrack3: no-symbols-control-file usr/lib/*
|
||||
|
||||
# Bundled, locally patched minizip (src/minizip): it adds a zipFlush() API the
|
||||
# system libminizip lacks (htscache.c flushes the cache .zip so an interrupted
|
||||
# crawl leaves a valid archive), plus Android/old-zlib portability fixes.
|
||||
libhttrack3: embedded-library *libminizip*
|
||||
3
debian/proxytrack.lintian-overrides
vendored
Normal file
3
debian/proxytrack.lintian-overrides
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
# Statically linked against httrack's bundled, patched minizip (see src/minizip
|
||||
# and libhttrack3's override): the zipFlush() API is absent from the system one.
|
||||
proxytrack: embedded-library *libminizip*
|
||||
2
debian/rules
vendored
2
debian/rules
vendored
@@ -135,7 +135,7 @@ binary-arch: build install
|
||||
dh_makeshlibs -a -X/usr/lib/$(DEB_HOST_MULTIARCH)/httrack/libtest --version-info
|
||||
dh_installdeb -a
|
||||
# we depend on the current version (ABI may change)
|
||||
dh_shlibdeps -a -ldebian/libhttrack2/usr/lib/$(DEB_HOST_MULTIARCH)
|
||||
dh_shlibdeps -a -ldebian/libhttrack3/usr/lib/$(DEB_HOST_MULTIARCH)
|
||||
dh_gencontrol -a
|
||||
dh_md5sums -a
|
||||
dh_builddeb -a
|
||||
|
||||
4
debian/source/lintian-overrides
vendored
4
debian/source/lintian-overrides
vendored
@@ -1,4 +1,6 @@
|
||||
httrack source: changelog-should-mention-nmu
|
||||
# Maintainer uploads sign the changelog as xavier@debian.org while the control
|
||||
# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
|
||||
httrack source: no-nmu-in-changelog
|
||||
httrack source: source-nmu-has-incorrect-version-number
|
||||
|
||||
# The bundled HTML pages are the genuine upstream documentation taken from
|
||||
|
||||
6
debian/upstream/metadata
vendored
Normal file
6
debian/upstream/metadata
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
Repository: https://github.com/xroche/httrack.git
|
||||
Repository-Browse: https://github.com/xroche/httrack
|
||||
Bug-Database: https://github.com/xroche/httrack/issues
|
||||
Bug-Submit: https://github.com/xroche/httrack/issues/new
|
||||
Contact: Xavier Roche <roche@httrack.com>
|
||||
58
fuzz/Makefile.am
Normal file
58
fuzz/Makefile.am
Normal file
@@ -0,0 +1,58 @@
|
||||
# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
|
||||
if FUZZERS
|
||||
noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
|
||||
fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx \
|
||||
fuzz-htsparse fuzz-singlefile fuzz-sitemap
|
||||
endif
|
||||
|
||||
AM_CPPFLAGS = \
|
||||
@DEFAULT_CFLAGS@ \
|
||||
@THREADS_CFLAGS@ \
|
||||
@V6_FLAG@ \
|
||||
@LFS_FLAG@ \
|
||||
-I$(top_srcdir)/src \
|
||||
-I$(top_srcdir)/src/coucal
|
||||
|
||||
# Static-link libhttrack.la: the internal symbols are hidden in the .so.
|
||||
AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
|
||||
LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
|
||||
|
||||
fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
|
||||
fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
|
||||
fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
|
||||
fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
|
||||
fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
|
||||
fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
|
||||
fuzz_url_SOURCES = fuzz-url.c fuzz.h
|
||||
fuzz_header_SOURCES = fuzz-header.c fuzz.h
|
||||
fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
|
||||
fuzz_htsparse_SOURCES = fuzz-htsparse.c fuzz.h
|
||||
fuzz_singlefile_SOURCES = fuzz-singlefile.c fuzz.h
|
||||
fuzz_sitemap_SOURCES = fuzz-sitemap.c fuzz.h
|
||||
|
||||
# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
|
||||
EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
|
||||
corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
|
||||
corpus/idna/idna.txt corpus/idna/unicode.txt \
|
||||
corpus/idna/regress-multilabel-leak.txt \
|
||||
corpus/entities/entities.txt \
|
||||
corpus/unescape/percent.txt \
|
||||
corpus/filters/filter.bin corpus/filters/filter-size.bin \
|
||||
corpus/filters/regress-empty-subject-unique.bin \
|
||||
corpus/filters/redos-star-classes.bin \
|
||||
corpus/filters/regress-classdepth-timeout.bin \
|
||||
corpus/url/http-url.txt corpus/url/relative-path.txt \
|
||||
corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
|
||||
corpus/header/full-response.txt corpus/header/redirect.txt \
|
||||
corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
|
||||
corpus/cachendx/regress-overadvance.bin \
|
||||
corpus/cachendx/regress-truncated-entry.bin \
|
||||
corpus/htsparse/basic.html corpus/htsparse/script-inscript.html \
|
||||
corpus/htsparse/meta-usemap.html corpus/htsparse/malformed.html \
|
||||
corpus/singlefile/img-src.html corpus/singlefile/link-rel.html \
|
||||
corpus/singlefile/style-block.html corpus/singlefile/style-attr.html \
|
||||
corpus/singlefile/srcset.html corpus/singlefile/rawtext.html \
|
||||
corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html \
|
||||
corpus/sitemap/urlset.xml corpus/sitemap/sitemapindex.xml \
|
||||
corpus/sitemap/truncated.xml corpus/sitemap/urlset.xml.gz
|
||||
15
fuzz/README.md
Normal file
15
fuzz/README.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# Fuzzing httrack
|
||||
|
||||
libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
|
||||
|
||||
```sh
|
||||
./bootstrap
|
||||
mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
|
||||
CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
|
||||
LDFLAGS="-fsanitize=address,undefined" \
|
||||
bash /path/to/httrack/configure --enable-fuzzers --disable-shared
|
||||
make
|
||||
bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60 # 60s per target
|
||||
```
|
||||
|
||||
Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.
|
||||
7
fuzz/corpus/cachendx/new-format.txt
Normal file
7
fuzz/corpus/cachendx/new-format.txt
Normal file
@@ -0,0 +1,7 @@
|
||||
8
|
||||
CACHE-1.1
|
||||
28
|
||||
Mon, 01 Jan 2024 00:00:00 GMT
|
||||
www.example.com
|
||||
/index.html
|
||||
123
|
||||
5
fuzz/corpus/cachendx/old-format.txt
Normal file
5
fuzz/corpus/cachendx/old-format.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
3
|
||||
1.0
|
||||
www.example.com
|
||||
/page
|
||||
5
|
||||
2
fuzz/corpus/cachendx/regress-overadvance.bin
Normal file
2
fuzz/corpus/cachendx/regress-overadvance.bin
Normal file
@@ -0,0 +1,2 @@
|
||||
32768
|
||||
CACHE-1.1
|
||||
5
fuzz/corpus/cachendx/regress-truncated-entry.bin
Normal file
5
fuzz/corpus/cachendx/regress-truncated-entry.bin
Normal file
@@ -0,0 +1,5 @@
|
||||
8
|
||||
CACHE-1.1
|
||||
1
|
||||
x
|
||||
www.example.com
|
||||
1
fuzz/corpus/charset/latin1.txt
Normal file
1
fuzz/corpus/charset/latin1.txt
Normal file
@@ -0,0 +1 @@
|
||||
café naďve ¤
|
||||
1
fuzz/corpus/charset/sjis.txt
Normal file
1
fuzz/corpus/charset/sjis.txt
Normal file
@@ -0,0 +1 @@
|
||||
コンピュータ
|
||||
BIN
fuzz/corpus/charset/utf8.txt
Normal file
BIN
fuzz/corpus/charset/utf8.txt
Normal file
Binary file not shown.
1
fuzz/corpus/entities/entities.txt
Normal file
1
fuzz/corpus/entities/entities.txt
Normal file
@@ -0,0 +1 @@
|
||||
&<>A☃é¬arealentity;�
|
||||
BIN
fuzz/corpus/filters/filter-size.bin
Normal file
BIN
fuzz/corpus/filters/filter-size.bin
Normal file
Binary file not shown.
BIN
fuzz/corpus/filters/filter.bin
Normal file
BIN
fuzz/corpus/filters/filter.bin
Normal file
Binary file not shown.
BIN
fuzz/corpus/filters/redos-star-classes.bin
Normal file
BIN
fuzz/corpus/filters/redos-star-classes.bin
Normal file
Binary file not shown.
BIN
fuzz/corpus/filters/regress-classdepth-timeout.bin
Normal file
BIN
fuzz/corpus/filters/regress-classdepth-timeout.bin
Normal file
Binary file not shown.
1
fuzz/corpus/filters/regress-empty-subject-unique.bin
Normal file
1
fuzz/corpus/filters/regress-empty-subject-unique.bin
Normal file
@@ -0,0 +1 @@
|
||||
**((
|
||||
10
fuzz/corpus/header/full-response.txt
Normal file
10
fuzz/corpus/header/full-response.txt
Normal file
@@ -0,0 +1,10 @@
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: text/html; charset=utf-8
|
||||
Content-Length: 1234
|
||||
Content-Encoding: gzip
|
||||
Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
|
||||
Etag: "abc"
|
||||
Location: http://example.com/x
|
||||
Set-Cookie: ID=42; path=/; domain=.example.com
|
||||
Content-Range: bytes 0-99/100
|
||||
Transfer-Encoding: chunked
|
||||
3
fuzz/corpus/header/redirect.txt
Normal file
3
fuzz/corpus/header/redirect.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
HTTP/1.0 301 Moved
|
||||
Location: /elsewhere
|
||||
Content-Disposition: attachment; filename="a.pdf"
|
||||
15
fuzz/corpus/htsparse/basic.html
Normal file
15
fuzz/corpus/htsparse/basic.html
Normal file
@@ -0,0 +1,15 @@
|
||||
<!DOCTYPE html>
|
||||
<html><head>
|
||||
<meta charset="utf-8">
|
||||
<base href="http://example.com/dir/">
|
||||
<title>Seed</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<script src="app.js"></script>
|
||||
</head><body>
|
||||
<h1>Hi</h1>
|
||||
<a href="page2.html">next</a>
|
||||
<a href="http://other.example.org/x?y=1#frag">abs</a>
|
||||
<img src="pic.png" srcset="a.png 1x, b.png 2x">
|
||||
<script>var u="inline.html"; document.write('<a href="gen.html">g</a>');</script>
|
||||
<form action="/cgi/submit"><input name="q"></form>
|
||||
</body></html>
|
||||
8
fuzz/corpus/htsparse/malformed.html
Normal file
8
fuzz/corpus/htsparse/malformed.html
Normal file
@@ -0,0 +1,8 @@
|
||||
<a href="unclosed.html
|
||||
<img src='mix"ed.png>
|
||||
<a href=noquote.html >bare</a>
|
||||
<!-- comment <a href="incomment.html"> -->
|
||||
<a href="tab newline
|
||||
.html">wsp</a>
|
||||
<script>unterminated "string and <a href=
|
||||
<a href=
|
||||
11
fuzz/corpus/htsparse/meta-usemap.html
Normal file
11
fuzz/corpus/htsparse/meta-usemap.html
Normal file
@@ -0,0 +1,11 @@
|
||||
<html><head>
|
||||
<meta http-equiv="refresh" content="0; url=redir.html">
|
||||
<base href="http://h/b/">
|
||||
<link rel="alternate" href="feed.xml">
|
||||
</head><body background="page-bg.jpg">
|
||||
<img src="map.png" usemap="#m">
|
||||
<map name="m"><area href="area1.html" coords="0,0,10,10"></map>
|
||||
<applet code="A.class" codebase="applets/"><param name="src" value="p.dat"></applet>
|
||||
<object data="o.swf"><embed src="e.svg"></object>
|
||||
<a href="http://ent/e.html">entity</a>
|
||||
</body></html>
|
||||
13
fuzz/corpus/htsparse/script-inscript.html
Normal file
13
fuzz/corpus/htsparse/script-inscript.html
Normal file
@@ -0,0 +1,13 @@
|
||||
<html><body>
|
||||
<script type="text/javascript">
|
||||
var a = "http://x/1.html", b = 'q\'uote', c = "/*not*/comment";
|
||||
// line "with" quotes and http://y/2.html
|
||||
/* block 'with' <a href="notparsed"> */
|
||||
var re = /a\/b\//g;
|
||||
document.write('<img src="w1.png">');
|
||||
document.writeln("<a href='w2.html'>k</a>");
|
||||
element.onclick = "location='onh.html'";
|
||||
</script>
|
||||
<a href="after.html" onmouseover="go('ev.html')">x</a>
|
||||
<div style="background:url(bg.png)">z</div>
|
||||
</body></html>
|
||||
1
fuzz/corpus/idna/idna.txt
Normal file
1
fuzz/corpus/idna/idna.txt
Normal file
@@ -0,0 +1 @@
|
||||
xn--bcher-kva.example.com
|
||||
1
fuzz/corpus/idna/regress-multilabel-leak.txt
Normal file
1
fuzz/corpus/idna/regress-multilabel-leak.txt
Normal file
@@ -0,0 +1 @@
|
||||
büchev.例åbücheple
|
||||
1
fuzz/corpus/idna/unicode.txt
Normal file
1
fuzz/corpus/idna/unicode.txt
Normal file
@@ -0,0 +1 @@
|
||||
bücher.例子.example
|
||||
1
fuzz/corpus/meta/meta-charset.html
Normal file
1
fuzz/corpus/meta/meta-charset.html
Normal file
@@ -0,0 +1 @@
|
||||
<html><head><meta charset="utf-8"></head><body>x</body></html>
|
||||
1
fuzz/corpus/meta/meta-content-first.html
Normal file
1
fuzz/corpus/meta/meta-content-first.html
Normal file
@@ -0,0 +1 @@
|
||||
<meta content="text/html; charset=gb2312" http-equiv="content-type">
|
||||
1
fuzz/corpus/meta/meta-http-equiv.html
Normal file
1
fuzz/corpus/meta/meta-http-equiv.html
Normal file
@@ -0,0 +1 @@
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
1
fuzz/corpus/meta/meta-truncated.html
Normal file
1
fuzz/corpus/meta/meta-truncated.html
Normal file
@@ -0,0 +1 @@
|
||||
<meta charset='utf-8
|
||||
1
fuzz/corpus/meta/meta-unquoted.html
Normal file
1
fuzz/corpus/meta/meta-unquoted.html
Normal file
@@ -0,0 +1 @@
|
||||
<meta charset = utf-8 />
|
||||
3
fuzz/corpus/singlefile/img-src.html
Normal file
3
fuzz/corpus/singlefile/img-src.html
Normal file
@@ -0,0 +1,3 @@
|
||||
<img src="a.png">
|
||||
<img src=big.png alt=over-cap>
|
||||
<img src="../escape.png"><img src="/abs.png"><img src="data:,x">
|
||||
4
fuzz/corpus/singlefile/link-rel.html
Normal file
4
fuzz/corpus/singlefile/link-rel.html
Normal file
@@ -0,0 +1,4 @@
|
||||
<link rel="stylesheet" href="s.css">
|
||||
<link rel=icon href=a.png>
|
||||
<link rel="next" href="p2.html">
|
||||
<link rel="preload" href="j.js">
|
||||
5
fuzz/corpus/singlefile/malformed.html
Normal file
5
fuzz/corpus/singlefile/malformed.html
Normal file
@@ -0,0 +1,5 @@
|
||||
<img src="unterminated.png
|
||||
<div style="background:url(a.png">
|
||||
<style>@import url(
|
||||
<!-- unterminated comment
|
||||
<a href=
|
||||
1
fuzz/corpus/singlefile/many-attrs.html
Normal file
1
fuzz/corpus/singlefile/many-attrs.html
Normal file
@@ -0,0 +1 @@
|
||||
<img src="a.png" a0="v" a1="v" a2="v" a3="v" a4="v" a5="v" a6="v" a7="v" a8="v" a9="v" a10="v" a11="v" a12="v" a13="v" a14="v" a15="v" a16="v" a17="v" a18="v" a19="v" a20="v" a21="v" a22="v" a23="v" a24="v" a25="v" a26="v" a27="v" a28="v" a29="v" a30="v" a31="v" a32="v" a33="v" a34="v" a35="v" a36="v" a37="v" a38="v" a39="v" a40="v" a41="v" a42="v" a43="v" a44="v" a45="v" a46="v" a47="v" a48="v" a49="v" a50="v" a51="v" a52="v" a53="v" a54="v" a55="v" a56="v" a57="v" a58="v" a59="v" a60="v" a61="v" a62="v" a63="v" a64="v" a65="v" a66="v" a67="v" a68="v" a69="v">
|
||||
4
fuzz/corpus/singlefile/rawtext.html
Normal file
4
fuzz/corpus/singlefile/rawtext.html
Normal file
@@ -0,0 +1,4 @@
|
||||
<script>var s="</scripting>"; if(a</b) x=1;</script>
|
||||
<script src="j.js"></script>
|
||||
<textarea></textareas></textarea>
|
||||
<title></titles></title>
|
||||
2
fuzz/corpus/singlefile/srcset.html
Normal file
2
fuzz/corpus/singlefile/srcset.html
Normal file
@@ -0,0 +1,2 @@
|
||||
<img srcset="a.png 1x, big.png 2x, a.png 100w">
|
||||
<source srcset="a.png,, a.png 2x," src="a.png">
|
||||
2
fuzz/corpus/singlefile/style-attr.html
Normal file
2
fuzz/corpus/singlefile/style-attr.html
Normal file
@@ -0,0 +1,2 @@
|
||||
<div style="background:url(a.png);list-style:url('a.png')"></div>
|
||||
<p style='background:url("a.png")'>x</p>
|
||||
5
fuzz/corpus/singlefile/style-block.html
Normal file
5
fuzz/corpus/singlefile/style-block.html
Normal file
@@ -0,0 +1,5 @@
|
||||
<style>@import "s.css";
|
||||
@import url(sub/b.css);
|
||||
div{background:url(a.png)}
|
||||
/* url(a.png) */ p:after{content:"url(a.png)"}
|
||||
</style>
|
||||
1
fuzz/corpus/sitemap/sitemapindex.xml
Normal file
1
fuzz/corpus/sitemap/sitemapindex.xml
Normal file
@@ -0,0 +1 @@
|
||||
<sitemapindex><sitemap><loc>http://h.test/s2.xml.gz</loc></sitemap></sitemapindex>
|
||||
1
fuzz/corpus/sitemap/truncated.xml
Normal file
1
fuzz/corpus/sitemap/truncated.xml
Normal file
@@ -0,0 +1 @@
|
||||
<urlset><loc>http://h.test/x
|
||||
1
fuzz/corpus/sitemap/urlset.xml
Normal file
1
fuzz/corpus/sitemap/urlset.xml
Normal file
@@ -0,0 +1 @@
|
||||
<?xml version="1.0"?><urlset><url><loc>http://h.test/a.html</loc></url><url><loc>https://h.test/b?x=1&y=2</loc></url></urlset>
|
||||
BIN
fuzz/corpus/sitemap/urlset.xml.gz
Normal file
BIN
fuzz/corpus/sitemap/urlset.xml.gz
Normal file
Binary file not shown.
1
fuzz/corpus/unescape/percent.txt
Normal file
1
fuzz/corpus/unescape/percent.txt
Normal file
@@ -0,0 +1 @@
|
||||
%41%zz%%20%c3%a9+%2e%2e%2f
|
||||
1
fuzz/corpus/url/http-url.txt
Normal file
1
fuzz/corpus/url/http-url.txt
Normal file
@@ -0,0 +1 @@
|
||||
http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag
|
||||
1
fuzz/corpus/url/regress-file-empty-path.txt
Normal file
1
fuzz/corpus/url/regress-file-empty-path.txt
Normal file
@@ -0,0 +1 @@
|
||||
file://
|
||||
1
fuzz/corpus/url/regress-long-path-abort.txt
Normal file
1
fuzz/corpus/url/regress-long-path-abort.txt
Normal file
@@ -0,0 +1 @@
|
||||
ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../
|
||||
1
fuzz/corpus/url/relative-path.txt
Normal file
1
fuzz/corpus/url/relative-path.txt
Normal file
@@ -0,0 +1 @@
|
||||
ftp://ftp.example.com/pub/../file.txt
|
||||
65
fuzz/fuzz-cachendx.c
Normal file
65
fuzz/fuzz-cachendx.c
Normal file
@@ -0,0 +1,65 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
|
||||
walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
|
||||
Mirrors the -#C cache-listing scan (htscoremain.c). */
|
||||
#include "fuzz.h"
|
||||
#include "htscache.h"
|
||||
#include "htslib.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *buf = fuzz_strdup(data, size);
|
||||
const char *const end = buf + size;
|
||||
char firstline[256];
|
||||
char *a = buf;
|
||||
|
||||
/* header: two length-prefixed fields (version, last-modified) */
|
||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
||||
|
||||
/* body: newline-delimited host/file/position triples; the length-prefixed
|
||||
scan must stay inside the buffer */
|
||||
while (a != NULL && a < end) {
|
||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
||||
char linepos[256];
|
||||
int pos;
|
||||
|
||||
a = strchr(a + 1, '\n');
|
||||
if (a == NULL)
|
||||
break;
|
||||
a++;
|
||||
a += cache_binput(a, end, line, HTS_URLMAXSIZE);
|
||||
a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
|
||||
a += cache_binput(a, end, linepos, 200);
|
||||
sscanf(linepos, "%d", &pos);
|
||||
(void) pos;
|
||||
}
|
||||
|
||||
freet(buf);
|
||||
return 0;
|
||||
}
|
||||
84
fuzz/fuzz-charset.c
Normal file
84
fuzz/fuzz-charset.c
Normal file
@@ -0,0 +1,84 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
|
||||
UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
|
||||
#include "fuzz.h"
|
||||
#include "htscharset.h"
|
||||
|
||||
static const char *const charsets[] = {
|
||||
"utf-8", "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
|
||||
"us-ascii", "shift_jis", "euc-jp", "iso-2022-jp", "gb2312",
|
||||
"big5", "euc-kr", "koi8-r", "utf-16", "unknown-charset",
|
||||
};
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
const char *charset;
|
||||
char *s;
|
||||
|
||||
if (size == 0)
|
||||
return 0;
|
||||
charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
|
||||
data++, size--;
|
||||
s = fuzz_strdup(data, size);
|
||||
|
||||
{
|
||||
char *utf8 = hts_convertStringToUTF8(s, size, charset);
|
||||
freet(utf8);
|
||||
}
|
||||
{
|
||||
char *enc = hts_convertStringFromUTF8(s, size, charset);
|
||||
freet(enc);
|
||||
}
|
||||
{
|
||||
size_t nChars = 0;
|
||||
hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
|
||||
|
||||
if (ucs != NULL) {
|
||||
char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
|
||||
freet(back);
|
||||
freet(ucs);
|
||||
}
|
||||
}
|
||||
{
|
||||
size_t i = 0;
|
||||
|
||||
while (i < size) {
|
||||
hts_UCS4 uc = 0;
|
||||
const size_t nr = hts_readUTF8(s + i, size - i, &uc);
|
||||
char out[8];
|
||||
|
||||
if (nr == 0)
|
||||
break;
|
||||
hts_writeUTF8(uc, out, sizeof(out));
|
||||
i += nr;
|
||||
}
|
||||
}
|
||||
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
51
fuzz/fuzz-entities.c
Normal file
51
fuzz/fuzz-entities.c
Normal file
@@ -0,0 +1,51 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
|
||||
destination size, so truncation bounds get exercised too. */
|
||||
#include "fuzz.h"
|
||||
#include "htsencoding.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static const size_t dsizes[] = {1, 2, 8, 64, 4096};
|
||||
size_t dsize;
|
||||
char *src, *dest;
|
||||
|
||||
if (size == 0)
|
||||
return 0;
|
||||
dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
|
||||
data++, size--;
|
||||
src = fuzz_strdup(data, size);
|
||||
dest = malloct(dsize);
|
||||
|
||||
(void) hts_unescapeEntities(src, dest, dsize);
|
||||
(void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
|
||||
|
||||
freet(dest);
|
||||
freet(src);
|
||||
return 0;
|
||||
}
|
||||
65
fuzz/fuzz-filters.c
Normal file
65
fuzz/fuzz-filters.c
Normal file
@@ -0,0 +1,65 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
|
||||
here). Input splits on the first NUL: pattern, then subject string. */
|
||||
#include "fuzz.h"
|
||||
#include "htsfilters.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *buf = fuzz_strdup(data, size);
|
||||
const char *joker = buf;
|
||||
const uint8_t *sep = memchr(data, '\0', size);
|
||||
/* subject in its own allocation so ASan bounds it apart from the pattern */
|
||||
char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
|
||||
: fuzz_strdup(data + size, 0);
|
||||
|
||||
(void) strjoker(nom, joker, NULL, NULL);
|
||||
{
|
||||
LLint sz = (LLint) size;
|
||||
int size_flag = 0;
|
||||
|
||||
(void) strjoker(nom, joker, &sz, &size_flag);
|
||||
}
|
||||
(void) strjokerfind(nom, joker);
|
||||
{
|
||||
char *filter = malloct(strlen(joker) + 2);
|
||||
char *filters[1];
|
||||
LLint sz = (LLint) size;
|
||||
int size_flag = 0, depth = 0;
|
||||
|
||||
filter[0] = '-';
|
||||
memcpy(filter + 1, joker, strlen(joker) + 1);
|
||||
filters[0] = filter;
|
||||
(void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
|
||||
freet(filter);
|
||||
}
|
||||
|
||||
freet(nom);
|
||||
freet(buf);
|
||||
return 0;
|
||||
}
|
||||
74
fuzz/fuzz-header.c
Normal file
74
fuzz/fuzz-header.c
Normal file
@@ -0,0 +1,74 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
|
||||
status line, treathead on each following header. Both consume raw bytes
|
||||
off the wire and copy fields into fixed htsblk buffers; treathead also
|
||||
mutates its line in place and drives cookie parsing. */
|
||||
#include "fuzz.h"
|
||||
#include "htslib.h"
|
||||
#include "htsbauth.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *buf = fuzz_strdup(data, size);
|
||||
htsblk r;
|
||||
t_cookie *cookie = calloct(1, sizeof(*cookie));
|
||||
char *line = malloct(size + 1);
|
||||
char *p = buf;
|
||||
int first = 1;
|
||||
|
||||
memset(&r, 0, sizeof(r));
|
||||
cookie->max_len = (int) sizeof(cookie->data);
|
||||
r.location = malloct(HTS_URLMAXSIZE * 2);
|
||||
r.location[0] = '\0';
|
||||
|
||||
/* feed one header line at a time, as the receive loop does */
|
||||
while (p != NULL && *p != '\0') {
|
||||
char *nl = strchr(p, '\n');
|
||||
size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
|
||||
size_t i, len = 0;
|
||||
|
||||
/* binput drops every '\r' on the wire; mirror it */
|
||||
for (i = 0; i < n; i++)
|
||||
if (p[i] != '\r')
|
||||
line[len++] = p[i];
|
||||
line[len] = '\0';
|
||||
if (first) {
|
||||
treatfirstline(&r, line);
|
||||
first = 0;
|
||||
} else {
|
||||
treathead(cookie, "www.example.com", "/", &r, line);
|
||||
}
|
||||
p = (nl != NULL) ? nl + 1 : NULL;
|
||||
}
|
||||
|
||||
freet(r.location);
|
||||
freet(line);
|
||||
freet(cookie);
|
||||
freet(buf);
|
||||
return 0;
|
||||
}
|
||||
175
fuzz/fuzz-htsparse.c
Normal file
175
fuzz/fuzz-htsparse.c
Normal file
@@ -0,0 +1,175 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the real htsparse() over a mocked engine: the minimal crawl state
|
||||
httpmirror() builds, then a page walked through the parser and discarded. The
|
||||
str/stre wiring below mirrors htsparse()'s call site in htscore.c; update it
|
||||
in lockstep if those structs gain a field the parser reads. */
|
||||
#include "fuzz.h"
|
||||
|
||||
#include "httrack-library.h"
|
||||
#include "htscore.h"
|
||||
#include "htsback.h"
|
||||
#include "htshash.h"
|
||||
#include "htsrobots.h"
|
||||
#include "htsparse.h"
|
||||
#include "htsmodules.h"
|
||||
#include "coucal.h"
|
||||
|
||||
/* htsparse ignores str.addLink on the internal parse; stub it. */
|
||||
static int fuzz_addlink(htsmoduleStruct *str, char *link) {
|
||||
(void) str;
|
||||
(void) link;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static int inited = 0;
|
||||
|
||||
httrackp *opt;
|
||||
cache_back cache;
|
||||
hash_struct hash;
|
||||
robots_wizard robots;
|
||||
struct_back *sback;
|
||||
char **filters = NULL;
|
||||
int filptr = 0;
|
||||
|
||||
htsblk r;
|
||||
htsmoduleStruct str;
|
||||
htsmoduleStructExtended stre;
|
||||
int ptr, error = 0, store_errpage = 0;
|
||||
int makeindex_done = 0, makeindex_links = 0;
|
||||
FILE *makeindex_fp = NULL;
|
||||
char makeindex_firstlink[HTS_URLMAXSIZE * 2] = "";
|
||||
LLint stat_fragment = 0, makestat_total = 0;
|
||||
int makestat_lnk = 0;
|
||||
char base[HTS_URLMAXSIZE * 2] = "";
|
||||
char codebase[HTS_URLMAXSIZE * 2] = "";
|
||||
char err_msg[1024] = "";
|
||||
|
||||
if (!inited) {
|
||||
hts_init();
|
||||
inited = 1;
|
||||
}
|
||||
|
||||
opt = hts_create_opt();
|
||||
opt->log = opt->errlog = NULL;
|
||||
opt->robots = 0;
|
||||
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 0; /* no on-disk cache */
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.cached_tests = coucal_new(0);
|
||||
coucal_value_is_malloc(cache.cached_tests, 1);
|
||||
|
||||
memset(&robots, 0, sizeof(robots));
|
||||
strcpybuff(robots.adr, "!");
|
||||
opt->robotsptr = &robots;
|
||||
|
||||
opt->maxfilter = maximum(opt->maxfilter, 128);
|
||||
filters_init(&filters, opt->maxfilter, 0);
|
||||
opt->filters.filters = &filters;
|
||||
opt->filters.filptr = &filptr;
|
||||
|
||||
opt->hash = &hash;
|
||||
hts_record_init(opt);
|
||||
hash_init(opt, &hash, opt->urlhack);
|
||||
hash.liens = (const lien_url *const *const *) &opt->liens;
|
||||
|
||||
sback = back_new(opt, opt->maxsoc * 32 + 1024);
|
||||
|
||||
/* ptr=1 (index 1 is the parsed page) selects full HTML parsing + the
|
||||
rewriter; urladr()/urlfil()/savename() alias heap(ptr), save=/dev/null. */
|
||||
hts_record_link(opt, "example.com", "/", "/dev/null", "", "", NULL);
|
||||
hts_record_link(opt, "example.com", "/index.html", "/dev/null", "", "", NULL);
|
||||
ptr = 1;
|
||||
|
||||
/* NUL-terminated in a size+1 alloc: htsparse one-past-reads onto the NUL. */
|
||||
hts_init_htsblk(&r);
|
||||
r.statuscode = 200;
|
||||
r.size = (LLint) size;
|
||||
r.adr = malloct(size + 1);
|
||||
if (size)
|
||||
memcpy(r.adr, data, size);
|
||||
r.adr[size] = '\0';
|
||||
strcpybuff(r.contenttype, "text/html");
|
||||
|
||||
memset(&str, 0, sizeof(str));
|
||||
memset(&stre, 0, sizeof(stre));
|
||||
str.err_msg = err_msg;
|
||||
str.filename = heap(ptr)->sav;
|
||||
str.mime = r.contenttype;
|
||||
str.url_host = heap(ptr)->adr;
|
||||
str.url_file = heap(ptr)->fil;
|
||||
str.size = (int) r.size;
|
||||
str.addLink = fuzz_addlink;
|
||||
str.opt = opt;
|
||||
str.sback = sback;
|
||||
str.cache = &cache;
|
||||
str.hashptr = &hash;
|
||||
str.numero_passe = 0;
|
||||
str.ptr_ = &ptr;
|
||||
str.page_charset_ = NULL;
|
||||
|
||||
stre.r_ = &r;
|
||||
stre.error_ = &error;
|
||||
stre.exit_xh_ = &opt->state.exit_xh;
|
||||
stre.store_errpage_ = &store_errpage;
|
||||
stre.base = base;
|
||||
stre.codebase = codebase;
|
||||
stre.filters_ = &filters;
|
||||
stre.filptr_ = &filptr;
|
||||
stre.robots_ = &robots;
|
||||
stre.hash_ = &hash;
|
||||
stre.makeindex_done_ = &makeindex_done;
|
||||
stre.makeindex_fp_ = &makeindex_fp;
|
||||
stre.makeindex_links_ = &makeindex_links;
|
||||
stre.makeindex_firstlink_ = makeindex_firstlink;
|
||||
stre.template_header_ = "";
|
||||
stre.template_body_ = "";
|
||||
stre.template_footer_ = "";
|
||||
stre.stat_fragment_ = &stat_fragment;
|
||||
stre.makestat_time = 0;
|
||||
stre.makestat_fp = NULL;
|
||||
stre.makestat_total_ = &makestat_total;
|
||||
stre.makestat_lnk_ = &makestat_lnk;
|
||||
stre.maketrack_fp = NULL;
|
||||
|
||||
(void) htsparse(&str, &stre);
|
||||
|
||||
freet(r.adr);
|
||||
back_delete_all(opt, &cache, sback);
|
||||
back_free(&sback);
|
||||
hash_free(&hash);
|
||||
coucal_delete(&cache.hashtable);
|
||||
coucal_delete(&cache.cached_tests);
|
||||
checkrobots_free(&robots);
|
||||
if (filters != NULL) {
|
||||
if (filters[0] != NULL)
|
||||
freet(filters[0]);
|
||||
freet(filters);
|
||||
}
|
||||
hts_record_free(opt);
|
||||
hts_free_opt(opt);
|
||||
return 0;
|
||||
}
|
||||
47
fuzz/fuzz-idna.c
Normal file
47
fuzz/fuzz-idna.c
Normal file
@@ -0,0 +1,47 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
|
||||
#include "fuzz.h"
|
||||
#include "htscharset.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *s = fuzz_strdup(data, size);
|
||||
|
||||
{
|
||||
char *idna = hts_convertStringUTF8ToIDNA(s, size);
|
||||
freet(idna);
|
||||
}
|
||||
{
|
||||
char *utf8 = hts_convertStringIDNAToUTF8(s, size);
|
||||
freet(utf8);
|
||||
}
|
||||
(void) hts_isStringIDNA(s, size);
|
||||
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
40
fuzz/fuzz-meta.c
Normal file
40
fuzz/fuzz-meta.c
Normal file
@@ -0,0 +1,40 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
|
||||
<meta> charset declaration. */
|
||||
#include "fuzz.h"
|
||||
#include "htscharset.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *html = fuzz_strdup(data, size);
|
||||
char *charset = hts_getCharsetFromMeta(html, size);
|
||||
|
||||
freet(charset);
|
||||
freet(html);
|
||||
return 0;
|
||||
}
|
||||
131
fuzz/fuzz-singlefile.c
Normal file
131
fuzz/fuzz-singlefile.c
Normal file
@@ -0,0 +1,131 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the --single-file rewriter (htssinglefile.c): hostile HTML walked
|
||||
through the tag, CSS url()/@import and srcset parsers, then re-serialized.
|
||||
The resolver is aimed at a private temp tree, so the inlining half (MIME
|
||||
guess, base64, nested stylesheet) is reached and nothing else on disk is. */
|
||||
#include "fuzz.h"
|
||||
|
||||
#include "httrack-library.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* Between a.png and big.png, so one input reaches both the inline path and the
|
||||
over-cap fallback. */
|
||||
#define FUZZ_SF_CAP 64
|
||||
|
||||
static char sf_root[512];
|
||||
static char sf_page[600];
|
||||
|
||||
/* The asset tree, in removal order: the subdirectory comes after its file. */
|
||||
static const char *const sf_files[] = {"a.png", "big.png", "j.js", "s.css",
|
||||
"sub/b.css", "sub", NULL};
|
||||
|
||||
static void sf_cleanup(void) {
|
||||
char path[700];
|
||||
int i;
|
||||
|
||||
for (i = 0; sf_files[i] != NULL; i++) {
|
||||
snprintf(path, sizeof(path), "%s/%s", sf_root, sf_files[i]);
|
||||
(void) remove(path);
|
||||
}
|
||||
(void) remove(sf_root);
|
||||
}
|
||||
|
||||
/* A missing asset would silently reduce the target to its parser half. */
|
||||
static void sf_write(const char *name, const char *data, size_t len) {
|
||||
char path[700];
|
||||
FILE *fp;
|
||||
|
||||
snprintf(path, sizeof(path), "%s/%s", sf_root, name);
|
||||
fp = fopen(path, "wb");
|
||||
if (fp == NULL || fwrite(data, 1, len, fp) != len)
|
||||
abort();
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
static void sf_text(const char *name, const char *data) {
|
||||
sf_write(name, data, strlen(data));
|
||||
}
|
||||
|
||||
static void sf_init(void) {
|
||||
static const char png[] = "\x89PNG\r\n\x1a\n";
|
||||
static const char big[4096] = "\x89PNG";
|
||||
const char *tmp = getenv("TMPDIR");
|
||||
char path[700];
|
||||
|
||||
hts_init();
|
||||
snprintf(sf_root, sizeof(sf_root), "%s/httrack-fuzz-sf-XXXXXX",
|
||||
tmp != NULL && tmp[0] != '\0' ? tmp : "/tmp");
|
||||
if (mkdtemp(sf_root) == NULL)
|
||||
abort();
|
||||
atexit(sf_cleanup);
|
||||
snprintf(sf_page, sizeof(sf_page), "%s/page.html", sf_root);
|
||||
snprintf(path, sizeof(path), "%s/sub", sf_root);
|
||||
if (mkdir(path, 0700) != 0)
|
||||
abort();
|
||||
sf_write("a.png", png, sizeof(png) - 1);
|
||||
sf_write("big.png", big, sizeof(big));
|
||||
sf_text("j.js", "var x=1;\n");
|
||||
/* @import plus a url(), so an inlined stylesheet recurses and its own
|
||||
relative reference is rebased. */
|
||||
sf_text("s.css", "@import url(sub/b.css);\ndiv{background:url(a.png)}\n");
|
||||
sf_text("sub/b.css", "p{background:url(../a.png)}\n");
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static int inited = 0;
|
||||
|
||||
String out = STRING_EMPTY;
|
||||
httrackp *opt;
|
||||
/* Exact-length, unterminated: the rewriter is span-based, so ASan bounds a
|
||||
read past html_len instead of it landing on a terminator. */
|
||||
char *html = malloct(size != 0 ? size : 1);
|
||||
|
||||
if (!inited) {
|
||||
sf_init();
|
||||
inited = 1;
|
||||
}
|
||||
memcpy(html, data, size);
|
||||
|
||||
opt = hts_create_opt();
|
||||
opt->log = opt->errlog = NULL;
|
||||
opt->single_file_max_size = FUZZ_SF_CAP;
|
||||
|
||||
StringClear(out);
|
||||
(void) singlefile_rewrite_html(opt, sf_root, sf_page, html, size,
|
||||
SINGLEFILE_MAX_PAGE_SIZE, &out);
|
||||
|
||||
StringFree(out);
|
||||
freet(html);
|
||||
hts_free_opt(opt);
|
||||
return 0;
|
||||
}
|
||||
60
fuzz/fuzz-sitemap.c
Normal file
60
fuzz/fuzz-sitemap.c
Normal file
@@ -0,0 +1,60 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the sitemap <loc> scanner (htssitemap.c): raw XML, gzip-framed bodies
|
||||
and truncated streams all arrive here straight off the network. */
|
||||
#include "fuzz.h"
|
||||
#include "htssitemap.h"
|
||||
|
||||
static hts_boolean sm_count(void *arg, const char *url) {
|
||||
int *const n = (int *) arg;
|
||||
|
||||
(void) url;
|
||||
(*n)++;
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static const int caps[] = {0, 1, 16, HTS_SITEMAP_MAX_URLS_DOC};
|
||||
hts_boolean is_index;
|
||||
char *body;
|
||||
int n = 0, cap;
|
||||
|
||||
if (size == 0)
|
||||
return 0;
|
||||
cap = caps[data[0] % (sizeof(caps) / sizeof(caps[0]))];
|
||||
data++, size--;
|
||||
/* A heap copy of exactly `size` bytes: the scanner must never rely on a
|
||||
terminator, and ASan turns any overread into a report. */
|
||||
body = malloct(size != 0 ? size : 1);
|
||||
memcpy(body, data, size);
|
||||
|
||||
(void) hts_sitemap_scan(body, size, cap, &is_index, sm_count, &n);
|
||||
|
||||
freet(body);
|
||||
return 0;
|
||||
}
|
||||
53
fuzz/fuzz-unescape.c
Normal file
53
fuzz/fuzz-unescape.c
Normal file
@@ -0,0 +1,53 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
|
||||
output buffer size, so the bounded-copy contract is exercised. */
|
||||
#include "fuzz.h"
|
||||
#include "httrack-library.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static const size_t bsizes[] = {1, 2, 16, 256, 8192};
|
||||
size_t bsize;
|
||||
char *s, *catbuff;
|
||||
|
||||
if (size == 0)
|
||||
return 0;
|
||||
bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
|
||||
data++, size--;
|
||||
s = fuzz_strdup(data, size);
|
||||
catbuff = malloct(bsize);
|
||||
|
||||
(void) unescape_http(catbuff, bsize, s);
|
||||
(void) unescape_http_unharm(catbuff, bsize, s, 0);
|
||||
(void) unescape_http_unharm(catbuff, bsize, s, 1);
|
||||
unescape_amp(s);
|
||||
|
||||
freet(catbuff);
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
49
fuzz/fuzz-url.c
Normal file
49
fuzz/fuzz-url.c
Normal file
@@ -0,0 +1,49 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
|
||||
is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
|
||||
in place. */
|
||||
#include "fuzz.h"
|
||||
#include "htscore.h"
|
||||
#include "htslib.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *s = fuzz_strdup(data, size);
|
||||
lien_adrfil *af = calloct(1, sizeof(*af));
|
||||
/* fil_simplifie rewrites in place and may grow an empty path to "./" */
|
||||
char *path = malloct(size + 3);
|
||||
|
||||
(void) ident_url_absolute(s, af);
|
||||
memcpy(path, s, size + 1);
|
||||
fil_simplifie(path);
|
||||
|
||||
freet(path);
|
||||
freet(af);
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
51
fuzz/fuzz.h
Normal file
51
fuzz/fuzz.h
Normal file
@@ -0,0 +1,51 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Shared helpers for the libFuzzer harnesses. */
|
||||
#ifndef FUZZ_H
|
||||
#define FUZZ_H
|
||||
|
||||
#define HTS_INTERNAL_BYTECODE
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "htsbase.h"
|
||||
|
||||
/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
|
||||
static char *fuzz_strdup(const uint8_t *data, size_t size) {
|
||||
char *s = malloct(size + 1);
|
||||
|
||||
memcpy(s, data, size);
|
||||
s[size] = '\0';
|
||||
return s;
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
|
||||
|
||||
#endif
|
||||
41
fuzz/run-fuzzers.sh
Executable file
41
fuzz/run-fuzzers.sh
Executable file
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# Drive every built harness against its seed corpus.
|
||||
# run-fuzzers.sh <build-fuzz-dir> check deterministic replay (CI smoke)
|
||||
# run-fuzzers.sh <build-fuzz-dir> [seconds] timed mutation run (discovery)
|
||||
# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
|
||||
# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
|
||||
set -euo pipefail
|
||||
|
||||
srcdir=$(cd "$(dirname "$0")" && pwd)
|
||||
bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
|
||||
mode=${2:-20}
|
||||
|
||||
status=0
|
||||
for f in "$bld"/fuzz-*; do
|
||||
if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
|
||||
case "$f" in *.o | *.c | *.dSYM) continue ;; esac
|
||||
name=$(basename "$f")
|
||||
corpus="$srcdir/corpus/${name#fuzz-}"
|
||||
if [ "$mode" = "check" ]; then
|
||||
echo "=== $name (replay) ==="
|
||||
[ -d "$corpus" ] || continue
|
||||
if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
|
||||
echo "*** $name FAILED on its corpus" >&2
|
||||
status=1
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
work=$(mktemp -d)
|
||||
args=("$work")
|
||||
[ -d "$corpus" ] && args+=("$corpus")
|
||||
echo "=== $name (${mode}s) ==="
|
||||
if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
|
||||
-artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
|
||||
echo "*** $name FAILED; artifacts:" >&2
|
||||
ls -l "$work" >&2
|
||||
status=1
|
||||
else
|
||||
rm -rf "$work"
|
||||
fi
|
||||
done
|
||||
exit $status
|
||||
@@ -5,7 +5,7 @@ Xavier Roche (xroche at httrack.com)
|
||||
project leader
|
||||
core engine, Windows/Linux GUI
|
||||
Yann Philippot (yphilippot at lemel.fr)
|
||||
for the java binary .class parser
|
||||
past contributor (java binary .class parser)
|
||||
|
||||
With the help of:
|
||||
Leto Kauler (molotov at tasmail.com)
|
||||
|
||||
130
history.txt
130
history.txt
@@ -4,6 +4,136 @@ HTTrack Website Copier release history:
|
||||
|
||||
This file lists all changes and fixes that have been made for HTTrack
|
||||
|
||||
3.49-15
|
||||
+ New: --single-file rewrites each saved page with its assets inlined as data: URIs (#713)
|
||||
+ New: --changes reports what a crawl added, updated or removed against the previous mirror (#714)
|
||||
+ New: --sitemap and --sitemap-url ingest sitemaps, so pages nothing links to are still found (#712)
|
||||
+ New: webhttrack exposes --warc-cdx, --wacz and --warc-max-size (#862)
|
||||
+ Fixed: --update and --purge-old destroyed a good local copy when the re-fetch got no response, was aborted mid-read, or when the backup meant to protect it failed (#746, #748, #758, #775)
|
||||
+ Fixed: an FTP re-fetch truncated the mirrored file, resumed a complete mirror with REST and spliced the old body into the new one, and a successful transfer was blanked when its backlog slot was swapped out (#771, #797, #798, #823)
|
||||
+ Fixed: a chunked response cut at a chunk boundary was stored and cached as complete (#840)
|
||||
+ Fixed: cache repair deleted the old cache before a rename it never checked, and the unlink-then-rename fallback could lose the destination (#779, #786, #790, #824)
|
||||
+ Fixed: several backward scans from strlen(s) - 1 read before their buffer on an empty string (#730, #768, #770, #814, #821)
|
||||
+ Fixed: a URL could be saved onto the engine's own temporary files, and the final path segment was never clamped (#774, #842, #852)
|
||||
+ Fixed: a query-string character reference the page charset cannot represent was left unescaped, changing how the query parses (#854)
|
||||
+ Fixed: a second --update pass overwrote the previous WARC and regenerated a page-less WACZ (#759)
|
||||
+ Fixed: WARC output dropped URLs of 1005 bytes or more, archived nothing for a 304 revisit, and marked an engine-forced not-modified as one the server sent (#778, #785, #826, #838, #839)
|
||||
+ Fixed: ProxyTrack overflowed its .arc header block, walked past the end of an .ndx buffer, trusted an unparsed offset, and crashed on a PROPFIND or on an entry with no usable Last-Modified (#793, #820, #825, #828)
|
||||
+ Fixed: webhttrack leaked the session id into crawled pages, overflowed its command line so a quoted value could inject flags, let a posted project path repoint the served root, and built its redirect Location in a 256-byte stack buffer (#700, #706, #707, #710)
|
||||
+ Fixed: in the web GUI, options ticked on by default could not be un-ticked, "max site size" set a per-file cap instead of the HTML one, and the mirror link on the finished page could not be followed (#708, #709, #725)
|
||||
+ Fixed: htsserver labelled every PNG as image/gif and offered JPEG as a download, and an unauthenticated GET of a directory spun the server forever (#724, #875)
|
||||
+ Fixed: webhttrack hung at exit when no mirror had been launched (#753)
|
||||
+ Fixed: oversized cache and header fields aborted the engine or overflowed a neighbouring field instead of being clipped (#701, #715, #717, #722, #732)
|
||||
+ Fixed: a -%S rules file of 4 GB or more overran the heap (#702)
|
||||
+ Fixed: the CLI display did not repaint when the terminal was resized (#97)
|
||||
+ Fixed: a document with no declared charset double-encoded the title lifted for the local index (#848)
|
||||
+ Fixed: a fragment on an inlined reference was dropped, losing an SVG sprite selector (#766)
|
||||
+ Fixed: several time helpers handed out libc's shared gmtime/localtime static rather than a reentrant breakdown (#794, #806)
|
||||
+ Changed: fatal-signal backtraces name engine frames instead of a bare module and offset (#705)
|
||||
+ Changed: --without-zlib is rejected at configure time rather than failing at link (#735)
|
||||
+ Changed: the offline documentation gains one GUI guide with screenshots, an Android option reference, and a restructured index
|
||||
+ Changed: multiple internal hardening, test and CI improvements
|
||||
|
||||
3.49-14
|
||||
+ New: WARC/1.1 archive output (--warc), with a sorted CDXJ index (--warc-cdx) and WACZ packaging (--wacz), also available from webhttrack (#668)
|
||||
+ New: -%F takes named footer fields such as {url}, {lastmodified}, {mime}, {charset} and {status} instead of a fixed layout (#667)
|
||||
+ New: a command-line guide organized by task ships with the offline documentation (#649)
|
||||
+ Fixed: on Windows, paths beyond MAX_PATH truncated files, and mirroring into a long or non-ASCII directory silently failed (#133)
|
||||
+ Fixed: the top index showed mojibake for non-ASCII project names and categories on Windows (#216)
|
||||
+ Fixed: webhttrack handed the engine the web form's charset rather than UTF-8, so a non-ASCII path mirrored into a mojibake directory (#629)
|
||||
+ Fixed: a non-ASCII single -O left the logs and the cache in a mangled twin directory on Windows (#630)
|
||||
+ Fixed: a path-ceiling truncation dropped the .delayed marker, losing the file (#623)
|
||||
+ Fixed: an oversized -%F footer aborted the crawl instead of being skipped (#669)
|
||||
+ Fixed: a rejected 206 resume could loop and lose the file rather than refetch it whole (#581)
|
||||
+ Fixed: default-port stripping was scheme-blind and dropped explicit ports from https and ftp URLs, and a :80 written with leading zeros mangled the host (#627, #638)
|
||||
+ Fixed: -K silently reset the -c socket count (#650)
|
||||
+ Fixed: signed-shift undefined behaviour in the zip-repair local-header read (#639)
|
||||
+ Changed: the offline documentation drops stale facts, gains an Android help page, and documents the filter wildcards and the real long option forms
|
||||
+ Changed: multiple internal hardening, test and CI improvements
|
||||
|
||||
3.49-13
|
||||
+ New: SOCKS5 proxy support, with scheme-aware -P URLs (socks5://, socks5h://, connect://) and plain HTTP tunneled through a CONNECT-only proxy (#563, #564)
|
||||
+ New: decode brotli and zstd content codings, advertised over TLS only as browsers do (#556)
|
||||
+ New: webhttrack exposes the engine options added since 3.49-2, among them --cookies-file, --pause and --strip-query (#587)
|
||||
+ Fixed: files of 2 GB or more were mishandled on Windows and on every 32-bit build (#569)
|
||||
+ Fixed: --update destroyed a good local copy when the re-fetch returned an HTTP error, was aborted by -M/-E, failed to decode, or came in short (#176, #521, #557, #562)
|
||||
+ Fixed: a self-redirect cookie wall was dropped instead of being re-fetched with the cookie (#15)
|
||||
+ Fixed: a stalled TLS handshake ignored --timeout, and synchronous DNS resolution could wedge a crawl past --max-time (#607, #613)
|
||||
+ Fixed: -M metered saved bytes rather than received volume, and overshot its cap under a slow server (#77, #520)
|
||||
+ Fixed: several network-facing overflows and denial-of-service paths in the Content-Range, chunked-transfer, cookie, filter and ProxyTrack cache parsers
|
||||
+ Fixed: a failed connect did not fall back to the next address on Windows (#579)
|
||||
+ Fixed: -P took an out-of-range port as a garbage port, and scanned past an IPv6 literal's closing bracket (#598, #602)
|
||||
+ Fixed: reject a port outside 1..65535 wherever one is parsed (a crawled URL, the htsserver and proxytrack listen arguments, an ftp:// URL), instead of letting a bare sscanf wrap a huge value into a plausible port and silently use it (#614, #626)
|
||||
+ Fixed: a configured proxy still resolved and dialed the origin itself (#592)
|
||||
+ Fixed: ~/ in the -O base path was never expanded (#270)
|
||||
+ Fixed: a non-ASCII -O output path was double-encoded on Windows once argv became UTF-8 (#621)
|
||||
+ Fixed: files under a non-ASCII project path were saved to a mangled directory on Windows (#217)
|
||||
+ Fixed: --build-top-index (-%i) and --protocol (-@i) were taken for the -i continue flag, wiping the URL list and exiting on the usage screen (#615)
|
||||
+ Fixed: webhttrack ignored LC_ALL/LC_MESSAGES and picked the wrong Chinese and Portuguese (#95)
|
||||
+ Fixed: webhttrack wrote its base path and httrack.ini to the filesystem root when $HOME was empty (#625)
|
||||
+ Fixed: crawls on a non-default port were slowed by a per-request pre-resolve (#181)
|
||||
+ Changed: Windows builds moved to Visual Studio 2022 and OpenSSL 3.x, the VS2008 project files are retired, and the binaries carry a version resource
|
||||
+ Changed: removed the obsolete Java-applet (.class) parser and the dead SWF module remnants
|
||||
+ Changed: multiple internal hardening, test and CI improvements (Windows and macOS crawl suites, HTML-parser fuzzing, parallel make check)
|
||||
|
||||
3.49-12
|
||||
+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
|
||||
+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
|
||||
+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
|
||||
+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
|
||||
+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
|
||||
+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
|
||||
+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
|
||||
+ Fixed: cookies.txt was created world-readable (#511)
|
||||
+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
|
||||
+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
|
||||
+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
|
||||
+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
|
||||
+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
|
||||
+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
|
||||
+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
|
||||
|
||||
3.49-11
|
||||
+ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
|
||||
+ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
|
||||
+ New: follow <source> and <track> media elements as embedded links (#451)
|
||||
+ New: added modern web MIME types to the type/extension table (#448)
|
||||
+ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
|
||||
+ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
|
||||
+ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
|
||||
+ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
|
||||
+ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
|
||||
+ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
|
||||
+ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
|
||||
+ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
|
||||
+ Changed: decode the full WHATWG set of HTML named character references (#443)
|
||||
+ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
|
||||
+ Changed: multiple internal hardening, build, test and CI improvements
|
||||
|
||||
3.49-10
|
||||
+ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
|
||||
+ New: --pause to space out file downloads by a random delay (#185)
|
||||
+ New: --strip-query to drop selected query keys from the dedup naming (#112)
|
||||
+ Changed: split the -%u URL hacks into independent --keep-www-prefix, --keep-double-slashes and --keep-query-order toggles (#271)
|
||||
+ Fixed: follow a redirect Location after dropping its #fragment, instead of requesting the fragment and polluting the saved name (#204)
|
||||
+ Fixed: escaped brackets inside a *[...] filter character class (#148)
|
||||
+ Fixed: honor the server's Content-Range when resuming a partial download, instead of appending overlapping bytes (#198)
|
||||
+ Fixed: abort the download as soon as the response type is excluded by -mime:, instead of fetching then discarding the body (#58)
|
||||
+ Fixed: keep size-based filter rules neutral until the file size is known (#143)
|
||||
+ Fixed: stop the mirror with a clean fatal error on a cache write failure, instead of crashing (#174, #219)
|
||||
+ Fixed: stop the 412/416 partial re-get loop on --continue and --update (#206)
|
||||
+ Fixed: keep an unrecognized URL tail instead of mangling it to .html (#115)
|
||||
+ Fixed: honor --tolerant (-%B) on a broken Content-Length, and fix an out-of-bounds read it exposed (#32, #41)
|
||||
+ Fixed: fall back to the next resolved address when a connection fails or stalls, instead of hanging on a dead IPv6 address
|
||||
+ Fixed: report why a -%L URL list could not be loaded (#49)
|
||||
+ Changed: multiple internal hardening, build and CI improvements
|
||||
|
||||
3.49-9
|
||||
+ Fixed: file-type detection from the Content-Type header: trust a declared type over a binary URL extension, honor --assume under the delayed type check, and keep a known extension against a bogus or empty Content-Type (#267, #29, #56)
|
||||
+ Fixed: an uninitialized-buffer read when the Content-Type is empty (#411)
|
||||
+ Fixed: restored C++ source-compatibility of the installed headers so reverse dependencies (httraqt) build again (#413)
|
||||
+ Changed: multiple internal build, packaging and test-harness improvements
|
||||
|
||||
3.49-8
|
||||
+ New: tunnel HTTPS downloads through the configured HTTP proxy via CONNECT (#85)
|
||||
+ New: parse every candidate URL in <img> and <source> srcset lists (#326)
|
||||
|
||||
@@ -19,7 +19,7 @@ MetaInfodir = $(datadir)/metainfo
|
||||
# Explicit filenames (e.g. ../history.txt, div/search.sh) resolve via VPATH and
|
||||
# need no prefix.
|
||||
HelpHtmlroot_DATA = ../httrack-doc.html ../history.txt
|
||||
HelpHtml_DATA = $(srcdir)/*.html
|
||||
HelpHtml_DATA = $(srcdir)/*.html $(srcdir)/*.js $(srcdir)/*.css
|
||||
HelpHtmldiv_DATA = div/search.sh
|
||||
HelpHtmlimg_DATA = $(srcdir)/img/*
|
||||
HelpHtmlimages_DATA = $(srcdir)/images/*
|
||||
@@ -39,12 +39,4 @@ MetaInfo_DATA = $(srcdir)/server/div/*.metainfo.xml
|
||||
EXTRA_DIST = $(HelpHtml_DATA) $(HelpHtmlimg_DATA) $(HelpHtmlimages_DATA) \
|
||||
$(HelpHtmldiv_DATA) $(WebHtml_DATA) $(WebHtmlimages_DATA) \
|
||||
$(WebPixmap_DATA) $(WebIcon16x16_DATA) $(WebIcon32x32_DATA) $(WebIcon48x48_DATA) \
|
||||
$(VFolderEntry_DATA) $(MetaInfo_DATA) \
|
||||
httrack.css
|
||||
|
||||
install-data-hook:
|
||||
if test ! -L $(DESTDIR)$(prefix)/share/httrack/html ; then \
|
||||
( cd $(DESTDIR)$(prefix)/share/httrack \
|
||||
&& $(LN_S) $(htmldir) html \
|
||||
) \
|
||||
fi
|
||||
$(VFolderEntry_DATA) $(MetaInfo_DATA)
|
||||
|
||||
241
html/abuse.html
241
html/abuse.html
@@ -1,111 +1,74 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
<!-- doc-chrome:head -->
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="How to mirror a site without hammering the server, and what a webmaster can do about crawlers that do.">
|
||||
<link rel="stylesheet" href="doc.css">
|
||||
<script src="doc.js" defer></script>
|
||||
<!-- /doc-chrome:head -->
|
||||
<title>Using HTTrack responsibly</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
<!-- doc-chrome:top -->
|
||||
<a class="skip" href="#main">Skip to content</a>
|
||||
|
||||
<h3>For HTTrack users:</h3>
|
||||
<header class="masthead">
|
||||
<img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier">
|
||||
<div class="tagline">Open Source offline browser</div>
|
||||
</header>
|
||||
|
||||
<div class="wrap">
|
||||
|
||||
<nav class="toc" aria-label="Documentation">
|
||||
<h2>On this page</h2>
|
||||
<ul>
|
||||
<li><a href="#for-httrack-users">For HTTrack users</a></li>
|
||||
<li><a href="#for-webmasters-having-problems-with-band">For webmasters having problems with bandwidth abuse / other abuses related to HTTrack</a></li>
|
||||
<li><a href="#please-follow-these-common-sense-rules-t">Please follow these common sense rules to avoid any network abuse</a></li>
|
||||
<li><a href="#how-to-limit-network-abuse-httrack-websi">How to limit network abuse HTTrack Website Copier FAQ (updated - DRAFT)</a></li>
|
||||
</ul>
|
||||
<h2>Start here</h2>
|
||||
<ul>
|
||||
<li><a href="guide.html">Interface guide</a></li>
|
||||
<li><a href="faq.html">FAQ and troubleshooting</a></li>
|
||||
<li><a href="abuse.html" aria-current="page" class="here">Using HTTrack responsibly</a></li>
|
||||
</ul>
|
||||
<h2>Command line</h2>
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line guide</a></li>
|
||||
<li><a href="httrack.man.html">Option reference</a></li>
|
||||
<li><a href="filters.html">Filter syntax</a></li>
|
||||
</ul>
|
||||
<h2>Developers</h2>
|
||||
<ul>
|
||||
<li><a href="dev.html">Programming</a></li>
|
||||
<li><a href="library.html">libhttrack API</a></li>
|
||||
<li><a href="plug.html">Callbacks</a></li>
|
||||
<li><a href="scripting.html">Scripting</a></li>
|
||||
<li><a href="cache.html">Cache format</a></li>
|
||||
<li><a href="changes.html">Change report format</a></li>
|
||||
</ul>
|
||||
<h2>More</h2>
|
||||
<ul>
|
||||
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
|
||||
<li><a href="contact.html">Contact and credits</a></li>
|
||||
<li><a href="index.html">All documentation</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main id="main">
|
||||
<!-- /doc-chrome:top -->
|
||||
|
||||
<h3 id="for-httrack-users">For HTTrack users:</h3>
|
||||
<ul>
|
||||
<li><a href="#USERS">Advice & what <u>not</u> to do when you are using HTTrack</a></li>
|
||||
</ul>
|
||||
<br>
|
||||
|
||||
<h3>For webmasters having problems with bandwidth abuse / other abuses related to HTTrack:</h3>
|
||||
<h3 id="for-webmasters-having-problems-with-band">For webmasters having problems with bandwidth abuse / other abuses related to HTTrack:</h3>
|
||||
<ul>
|
||||
<li><a href="#WEBMASTERS">Abuse FAQ for <u>webmasters</a></u></li>
|
||||
</ul>
|
||||
@@ -121,9 +84,9 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
|
||||
|
||||
<h2 align="center"><em>Advice & what <u>not</u> to do</em></h2>
|
||||
<h1>Advice & what <u>not</u> to do</h1>
|
||||
|
||||
<center><h3>Please follow these <i>common sense</i> rules to avoid any network abuse</h3></center>
|
||||
<center><h3 id="please-follow-these-common-sense-rules-t">Please follow these <i>common sense</i> rules to avoid any network abuse</h3></center>
|
||||
|
||||
<br>
|
||||
|
||||
@@ -178,9 +141,9 @@ Downloading a site can overload it, if you have a fast pipe, or if you capture t
|
||||
|
||||
|
||||
|
||||
<h2 align="center"><em>Abuse FAQ for <u>webmasters</u></em></h2>
|
||||
<h2 id="abuse-faq-for-webmasters" align="center"><em>Abuse FAQ for <u>webmasters</u></em></h2>
|
||||
|
||||
<center><h3>How to limit network abuse
|
||||
<center><h3 id="how-to-limit-network-abuse-httrack-websi">How to limit network abuse
|
||||
<br>
|
||||
HTTrack Website Copier FAQ (updated - DRAFT)
|
||||
</h3></center>
|
||||
@@ -217,7 +180,7 @@ school or
|
||||
shows. They might do that because they are connected through expensive modem connection,
|
||||
or because they would like to consult pages while travelling, or archive sites that may be
|
||||
removed
|
||||
one day, make some data mining, comiling information ("if only I could find this
|
||||
one day, make some data mining, compiling information ("if only I could find this
|
||||
website I saw one day.."). <br>
|
||||
There are many good reasons to mirror websites, and this helps many good people.<br>
|
||||
As a webmaster, you might be interested to use such tools, too: test broken links, move a
|
||||
@@ -229,7 +192,7 @@ test the webserver response and performances, index it..<br>
|
||||
Anyway, bandwidth abuse can be a problem. If your site is regularly "clobbered"
|
||||
by evil downloaders, you have <br>
|
||||
various solutions. You have radical solutions, and intermediate solutions. I strongly
|
||||
recomment not to use<br>
|
||||
recommend not to use<br>
|
||||
radical solutions, because of the previous remarks (good people often mirror websites).<br>
|
||||
<br>
|
||||
In general, for all solutions,<br>
|
||||
@@ -244,7 +207,7 @@ or, to be extreme: if you unplug the wire, there will be no bandwidth abuse<br>
|
||||
Good: Will work with good people. Many good people just don't KNOW that they can slow down
|
||||
a network.<br>
|
||||
Bad: Will **only** work with good people<br>
|
||||
How to do: Obvious - place a note, a warning, an article, a draw, a poeme or whatever you
|
||||
How to do: Obvious - place a note, a warning, an article, a draw, a poem or whatever you
|
||||
want<br>
|
||||
<br>
|
||||
</li><li>Use "robots.txt" file<br>
|
||||
@@ -266,7 +229,7 @@ Good: Efficient<br>
|
||||
Bad: Multiple users behind proxies will be slow down, not really easy to setup<br>
|
||||
How to do: Depends on webserver. Might be done with low-level IP rules (QoS)<br>
|
||||
<br>
|
||||
</li><li>Priorize small files, against large files<br>
|
||||
</li><li>Prioritize small files, against large files<br>
|
||||
Good: Efficient if large files are the cause of abuse<br>
|
||||
Bad: Not always efficient<br>
|
||||
How to do: Depends on the webserver<br>
|
||||
@@ -283,7 +246,7 @@ How to do: Use routine QoS (fair queuing), or webserver options<br>
|
||||
<br>
|
||||
</li><li>Use technical tricks (like javascript) to hide URLs<br>
|
||||
Good: Efficient<br>
|
||||
Bad: The most efficient tricks will also cause your website to he heavy, and not
|
||||
Bad: The most efficient tricks will also cause your website to be heavy, and not
|
||||
user-friendly (and therefore less attractive, even for surfing users). Remember: clients
|
||||
or visitors might want to consult offline your website. Advanced users will also be still
|
||||
able to note the URLs and catch them. Will not work on non-javascript browsers. It will
|
||||
@@ -335,22 +298,22 @@ Example: Use things like
|
||||
</li><li>Use technical tricks to temporarily ban IPs<br>
|
||||
Good: Efficient<br>
|
||||
Bad: Radical (your site will only be available online for all users), not easy to setup<br>
|
||||
How to to: Create fake links with "killing" targets<br>
|
||||
How to do: Create fake links with "killing" targets<br>
|
||||
Example: Use things like <a href="killme.cgi"><nothing></a>
|
||||
(again an example in php:)<br>
|
||||
<tt>
|
||||
<tt>
|
||||
<pre>
|
||||
<?php
|
||||
// Add IP.
|
||||
add_temp_firewall_rule($REMOTE_ADDR,"30s");
|
||||
?>
|
||||
function add_temp_firewall_rule($addr) {
|
||||
// The chain chhttp is flushed in a cron job to avoid ipchains overflow
|
||||
system("/usr/bin/sudo -u root /sbin/ipchains -I 1 chhttp -p tcp -s ".$addr." --dport 80 -j REJECT");
|
||||
syslog("user rejected due to too many copy attemps : ".$addr);
|
||||
}
|
||||
|
||||
<pre>
|
||||
<?php
|
||||
// Add IP.
|
||||
add_temp_firewall_rule($REMOTE_ADDR,"30s");
|
||||
?>
|
||||
function add_temp_firewall_rule($addr) {
|
||||
// The chain chhttp is flushed in a cron job to avoid ipchains overflow
|
||||
system("/usr/bin/sudo -u root /sbin/ipchains -I 1 chhttp -p tcp -s ".$addr." --dport 80 -j REJECT");
|
||||
syslog("user rejected due to too many copy attemps : ".$addr);
|
||||
}
|
||||
|
||||
<pre>
|
||||
|
||||
</tt>
|
||||
<br>
|
||||
@@ -409,15 +372,15 @@ Example:<br>
|
||||
FOS('mycompany.com','smith?subject=Hi, John','Click here to email me!')<br>
|
||||
// --><br>
|
||||
</script><br>
|
||||
<noscript><br>
|
||||
smith at mycompany dot com<br>
|
||||
</noscript><br>
|
||||
<noscript><br>
|
||||
smith at mycompany dot com<br>
|
||||
</noscript><br>
|
||||
</tt>
|
||||
<br>
|
||||
</li><li>Another one is to create images of emails<br>
|
||||
Good: Efficient, does not require javascript<br>
|
||||
Bad: There is still the problem of the link (mailto:), images are bigger than text, and it can cause problems for blind people (a good solution is use an ALT attribute with the email written like "smith at mycompany dot com")<br>
|
||||
How to do: Not so obvious of you do not want to create images by yourself<br>
|
||||
How to do: Not so obvious if you do not want to create images by yourself<br>
|
||||
Example: (php, Unix)<br>
|
||||
|
||||
<tt>
|
||||
@@ -491,7 +454,7 @@ echo <br>
|
||||
</li><li>You can also create temporary email aliases, each week, for all users<br>
|
||||
Good: Efficient, and you can give your real email in your reply-to address<br>
|
||||
Bad: Temporary emails<br>
|
||||
How to do: Not so hard todo<br>
|
||||
How to do: Not so hard to do<br>
|
||||
Example: (script & php, Unix)<br>
|
||||
|
||||
<tt>
|
||||
@@ -566,26 +529,14 @@ And then, put the email address in your pages through:
|
||||
|
||||
<br>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<!-- doc-chrome:bottom -->
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
|
||||
|
||||
<footer>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</footer>
|
||||
<!-- /doc-chrome:bottom -->
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
|
||||
|
||||
|
||||
159
html/addurl.html
159
html/addurl.html
@@ -1,155 +1,10 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<title>Moved to the HTTrack interface guide</title>
|
||||
<link rel="canonical" href="guide.html#step-address">
|
||||
<meta http-equiv="refresh" content="0; url=guide.html#step-address">
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>Add a URL</em></h2>
|
||||
|
||||
<br>
|
||||
|
||||
<ol>
|
||||
<li>Enter a typical Web address</li>
|
||||
<br><small>Just type in your address in the field</small><br><br>
|
||||
<center><img src="img/addurl1.gif" border="0"></center>
|
||||
<br><br>OR<br><br>
|
||||
<li>Enter a Web address with authentication</li>
|
||||
<br><small>Useful when you need basic authentication to watch the Web page</small><br><br>
|
||||
<center><img src="img/addurl2.gif" border="0"></center>
|
||||
<br><br>OR<br><br>
|
||||
<li>Capture a link from your Web browser to HTTrack</li>
|
||||
<br><small>Use this tool only for form-based pages (pages delivered after submiting a form) that need some analysis</small><br><br>
|
||||
<center><img src="img/addurl3.gif" border="0"></center>
|
||||
<br>
|
||||
<br>
|
||||
<small>Set, as explained, your Web browser proxy preferences to the values indicated : set the proxy's address, and the proxy's port,
|
||||
then click on the button or link as you usually do in your Web browser.
|
||||
<br>The temporary proxy, installed by HTTrack, will then
|
||||
capture the link and display a confirmation page.
|
||||
<br><br>
|
||||
</small>
|
||||
<center><img src="img/addurl4.gif" border="0"></center>
|
||||
<br>
|
||||
<center><img src="img/addurl5.gif" border="0"></center>
|
||||
</ol>
|
||||
|
||||
<br><br><br><br>
|
||||
<p align="right">Back to <a href="index.html">Home</a></p>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
<body><p>Adding a URL is now part of <a href="guide.html#step-address">the HTTrack interface guide</a>.</p></body>
|
||||
</html>
|
||||
|
||||
|
||||
|
||||
11
html/android.html
Normal file
11
html/android.html
Normal file
@@ -0,0 +1,11 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Moved to the HTTrack interface guide</title>
|
||||
<link rel="canonical" href="guide.html">
|
||||
<meta http-equiv="refresh" content="0; url=guide.html#droid/">
|
||||
</head>
|
||||
<body><p>This page is now part of
|
||||
<a href="guide.html#droid/">the HTTrack interface guide</a>.</p></body>
|
||||
</html>
|
||||
503
html/cache.html
503
html/cache.html
@@ -1,293 +1,238 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Cache format specification</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
<!-- doc-chrome:head -->
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="The format of the HTTrack cache in hts-cache, and how to read it.">
|
||||
<link rel="stylesheet" href="doc.css">
|
||||
<script src="doc.js" defer></script>
|
||||
<!-- /doc-chrome:head -->
|
||||
<title>HTTrack cache format</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
<!-- doc-chrome:top -->
|
||||
<a class="skip" href="#main">Skip to content</a>
|
||||
|
||||
<h2 align="center"><em>Cache format specification</em></h2>
|
||||
<header class="masthead">
|
||||
<img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier">
|
||||
<div class="tagline">Open Source offline browser</div>
|
||||
</header>
|
||||
|
||||
<div class="wrap">
|
||||
|
||||
<nav class="toc" aria-label="Documentation">
|
||||
<h2>Start here</h2>
|
||||
<ul>
|
||||
<li><a href="guide.html">Interface guide</a></li>
|
||||
<li><a href="faq.html">FAQ and troubleshooting</a></li>
|
||||
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
|
||||
</ul>
|
||||
<h2>Command line</h2>
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line guide</a></li>
|
||||
<li><a href="httrack.man.html">Option reference</a></li>
|
||||
<li><a href="filters.html">Filter syntax</a></li>
|
||||
</ul>
|
||||
<h2>Developers</h2>
|
||||
<ul>
|
||||
<li><a href="dev.html">Programming</a></li>
|
||||
<li><a href="library.html">libhttrack API</a></li>
|
||||
<li><a href="plug.html">Callbacks</a></li>
|
||||
<li><a href="scripting.html">Scripting</a></li>
|
||||
<li><a href="cache.html" aria-current="page" class="here">Cache format</a></li>
|
||||
<li><a href="changes.html">Change report format</a></li>
|
||||
</ul>
|
||||
<h2>More</h2>
|
||||
<ul>
|
||||
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
|
||||
<li><a href="contact.html">Contact and credits</a></li>
|
||||
<li><a href="index.html">All documentation</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main id="main">
|
||||
<!-- /doc-chrome:top -->
|
||||
|
||||
<h1>Cache format specification</h1>
|
||||
|
||||
<br />
|
||||
|
||||
For updating purpose, HTTrack stores original (untouched) HTML data,
|
||||
references to downloaded files, and other meta-data (especially parts of the HTTP headers) in a cache,
|
||||
located in the hts-cache directory. Because local html pages are always modified to "fit" the local
|
||||
filesystem structure, and because meta-data such as the last-Modified date and Etag can not be stored
|
||||
with the associated files, the cache is absolutely mandatory for reprocessing (update/continue) phases.
|
||||
|
||||
<br /><br />
|
||||
|
||||
<h3>The (new) cache.zip format</h3>
|
||||
|
||||
The 3.31 release of HTTrack introduces a new cache format, more extensible and efficient than the previous one (ndx/dat format).
|
||||
|
||||
The main advantages of this cache are:
|
||||
|
||||
<ul>
|
||||
<li>One single file for a complete website cache archive</li>
|
||||
<li>Standard <a href="http://www.pkware.com/products/enterprise/white_papers/appnote.txt" target="_new">ZIP</a> format, that can be easily reused on most platforms and languages</li>
|
||||
<li>Compressed data with the efficient and opened <a href="http://www.gzip.org/zlib/" target="_new">zlib</a> format</li>
|
||||
</ul>
|
||||
|
||||
The cache is made of ZIP files entries ; with one ZIP file entry per fetched URL (successfully or not - errors are also stored).<br />
|
||||
For each entry:
|
||||
<ul>
|
||||
<li>The ZIP file name is the original URL [<small><a href="#orig">see notes below</a></small>]</li>
|
||||
<li>The ZIP file contents, <b>if available</b>, is the original (compressed, using the deflate algorythm) data</li>
|
||||
<li>The ZIP file extra field (in the local file header) contains a list of meta-fields, very similar to the <a href="http://www.ietf.org/rfc/rfc2616.txt?number=2616" target="new_">HTTP</a> headers fields. See also <a href="http://www.ietf.org/rfc/rfc2396.txt?number=2396" target="new_">RFC</a>.</li><br />
|
||||
<li>The ZIP file timestamp follows the "Last-Modified-Since" field given for this URL, if any</li>
|
||||
</ul>
|
||||
|
||||
Example of cache file:
|
||||
<br />
|
||||
|
||||
<pre>
|
||||
$ unzip -l hts-cache/new.zip
|
||||
Archive: hts-cache/new.zip
|
||||
HTTrack Website Copier/3.31-ALPHA-4 mirror complete in 3 seconds : 5 links scanned,
|
||||
3 files written (16109 bytes overall) [17690 bytes received at 5896 bytes/sec]
|
||||
(1 errors, 0 warnings, 0 messages)
|
||||
Length Date Time Name
|
||||
-------- ---- ---- ----
|
||||
94 07-18-03 08:59 http://www.httrack.com/robots.txt
|
||||
9866 01-17-04 01:09 http://www.httrack.com/html/cache.html
|
||||
0 05-11-03 13:31 http://www.httrack.com/html/images/bg_rings.gif
|
||||
207 01-19-04 05:49 http://www.httrack.com/html/fade.gif
|
||||
0 05-11-03 13:31 http://www.httrack.com/html/images/header_title_4.gif
|
||||
-------- -------
|
||||
10167 5 files
|
||||
</pre>
|
||||
|
||||
Example of cache file meta-data:
|
||||
<br />
|
||||
|
||||
<pre>
|
||||
HTTP/1.1 200 OK
|
||||
X-In-Cache: 1
|
||||
X-StatusCode: 200
|
||||
X-StatusMessage: OK
|
||||
X-Size: 94
|
||||
Content-Type: text/plain
|
||||
Last-Modified: Fri, 18 Jul 2003 08:59:11 GMT
|
||||
Etag: "40ebb5-5e-3f17b6df"
|
||||
X-Addr: www.httrack.com
|
||||
X-Fil: /robots.txt
|
||||
</pre>
|
||||
|
||||
There are also specific issues regarding this format:
|
||||
|
||||
<ul>
|
||||
<li>The data in the central directory (such as CD extra field, and CD comments) are not used</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^16 files (65535) ; in such case the total number of entries in the 32-bit central directory is 65536 (0xffff), but the presence of the 64-bit central directory is not mandatory</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory must be present <b>(not currently supported)</b></li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
<b>Meta-data stored in the "extra field" of the local file headers</b><br />
|
||||
|
||||
The extra field is composed of text data, and this text data is composed of distinct lines of headers.
|
||||
The end of text, <b>or</b> a double CR/LF, mark the end of this zone.
|
||||
This method allows you to optionally store original HTTP headers just after the "meta-data" headers for informational use.<br />
|
||||
|
||||
<br />
|
||||
<b>The status line (the first headers line)</b><br />
|
||||
|
||||
Status-Line = HTTP-Version SP Status-Code SP X-Reason-Phrase CRLF<br />
|
||||
|
||||
<br />
|
||||
<b>Other lines:</b><br />
|
||||
|
||||
<br />
|
||||
<b>Specific fields:</b><br />
|
||||
<ul>
|
||||
<li>X-In-Cache</li><br>
|
||||
Indicates if the data are present (value=1) in the cache (that is, as ZIP data), or in an external file (value=0).
|
||||
This field MUST be the first field.
|
||||
|
||||
<li>X-StatusCode</li><br>
|
||||
The modified (by httrack) status code after processing. 304 error codes ("Not modified"), for example, are transformed into "200" codes after processing.
|
||||
|
||||
<li>X-StatusMessage</li><br>
|
||||
The modified (by httrack) status message.
|
||||
|
||||
<li>X-Size</li><br>
|
||||
The stored (either in cache, or in an external file) data size.
|
||||
|
||||
<li>X-Charset</li><br>
|
||||
The original charset.
|
||||
|
||||
<li>X-Addr</li><br>
|
||||
The original URL address part.
|
||||
|
||||
<li>X-Fil</li><br>
|
||||
The original URL path part.
|
||||
|
||||
<li>X-Save</li><br>
|
||||
The local filename, depending on user's "build structure" preferences.
|
||||
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
<b>Standard (RFC 2616) "useful" fields:</b><br />
|
||||
<ul>
|
||||
<li>Content-Type</li>
|
||||
<li>Last-Modified</li>
|
||||
<li>Etag</li>
|
||||
<li>Location</li>
|
||||
<li>Content-Disposition</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
<b>Specific fields in "BNF-like" grammar:</b><br />
|
||||
|
||||
<pre>
|
||||
X-In-Cache = "X-In-Cache" ":" 1*DIGIT
|
||||
X-StatusCode = "X-StatusCode" ":" 1*DIGIT
|
||||
X-StatusMessage = "X-StatusMessage" ":" *<TEXT, excluding CR, LF>
|
||||
X-Size = "X-Size" ":" 1*DIGIT
|
||||
X-Charset = "X-Charset" ":" value
|
||||
X-Addr = "X-Addr" ":" scheme ":" "//" authority
|
||||
X-Fil = "X-Fil" ":" rel_path
|
||||
X-Save = "X-Save" ":" rel_path
|
||||
</pre>
|
||||
|
||||
<b>RFC standard fields:</b><br />
|
||||
|
||||
<pre>
|
||||
Content-Type = "Content-Type" ":" media-type
|
||||
Last-Modified = "Last-Modified" ":" HTTP-date
|
||||
Etag = "ETag" ":" entity-tag
|
||||
Location = "Location" ":" absoluteURI
|
||||
Content-Disposition = "Content-Disposition" ":" disposition-type *( ";" disposition-parm )
|
||||
</pre>
|
||||
|
||||
<br />
|
||||
And, for your information,
|
||||
<pre>
|
||||
X-Reason-Phrase = *<TEXT, with a maximum of 32 characters, and excluding CR, LF>
|
||||
</pre>
|
||||
|
||||
|
||||
<a name="orig">Note: Because the URLs may have an unexpected format, especially with double "/" inside, and other reserved characters ("?", "&" ..),
|
||||
various ZIP uncompressors can potentially have troubles accessing or decompressing the data.
|
||||
Libraries should generally handle this peculiar format, however.
|
||||
</a>
|
||||
For updating purpose, HTTrack stores original (untouched) HTML data,
|
||||
references to downloaded files, and other meta-data (especially parts of the HTTP headers) in a cache,
|
||||
located in the hts-cache directory. Because local html pages are always modified to "fit" the local
|
||||
filesystem structure, and because meta-data such as the last-Modified date and Etag can not be stored
|
||||
with the associated files, the cache is absolutely mandatory for reprocessing (update/continue) phases.
|
||||
|
||||
<br /><br />
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<h3 id="the-new-cache-zip-format">The (new) cache.zip format</h3>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
The 3.31 release of HTTrack introduces a new cache format, more extensible and efficient than the previous one (ndx/dat format).
|
||||
|
||||
The main advantages of this cache are:
|
||||
|
||||
<ul>
|
||||
<li>One single file for a complete website cache archive</li>
|
||||
<li>Standard <a href="http://www.pkware.com/products/enterprise/white_papers/appnote.txt" target="_new">ZIP</a> format, that can be easily reused on most platforms and languages</li>
|
||||
<li>Compressed data with the efficient and opened <a href="http://www.gzip.org/zlib/" target="_new">zlib</a> format</li>
|
||||
</ul>
|
||||
|
||||
The cache is made of ZIP files entries ; with one ZIP file entry per fetched URL (successfully or not - errors are also stored).<br />
|
||||
For each entry:
|
||||
<ul>
|
||||
<li>The ZIP file name is the original URL [<small><a href="#orig">see notes below</a></small>]</li>
|
||||
<li>The ZIP file contents, <b>if available</b>, is the original (compressed, using the deflate algorythm) data</li>
|
||||
<li>The ZIP file extra field (in the local file header) contains a list of meta-fields, very similar to the <a href="http://www.ietf.org/rfc/rfc2616.txt?number=2616" target="new_">HTTP</a> headers fields. See also <a href="http://www.ietf.org/rfc/rfc2396.txt?number=2396" target="new_">RFC</a>.</li><br />
|
||||
<li>The ZIP file timestamp follows the "Last-Modified-Since" field given for this URL, if any</li>
|
||||
</ul>
|
||||
|
||||
Example of cache file:
|
||||
<br />
|
||||
|
||||
<pre>
|
||||
$ unzip -l hts-cache/new.zip
|
||||
Archive: hts-cache/new.zip
|
||||
HTTrack Website Copier/3.31-ALPHA-4 mirror complete in 3 seconds : 5 links scanned,
|
||||
3 files written (16109 bytes overall) [17690 bytes received at 5896 bytes/sec]
|
||||
(1 errors, 0 warnings, 0 messages)
|
||||
Length Date Time Name
|
||||
-------- ---- ---- ----
|
||||
94 07-18-03 08:59 http://www.httrack.com/robots.txt
|
||||
9866 01-17-04 01:09 http://www.httrack.com/html/cache.html
|
||||
0 05-11-03 13:31 http://www.httrack.com/html/images/bg_rings.gif
|
||||
207 01-19-04 05:49 http://www.httrack.com/html/fade.gif
|
||||
0 05-11-03 13:31 http://www.httrack.com/html/images/header_title_4.gif
|
||||
-------- -------
|
||||
10167 5 files
|
||||
</pre>
|
||||
|
||||
Example of cache file meta-data:
|
||||
<br />
|
||||
|
||||
<pre>
|
||||
HTTP/1.1 200 OK
|
||||
X-In-Cache: 1
|
||||
X-StatusCode: 200
|
||||
X-StatusMessage: OK
|
||||
X-Size: 94
|
||||
Content-Type: text/plain
|
||||
Last-Modified: Fri, 18 Jul 2003 08:59:11 GMT
|
||||
Etag: "40ebb5-5e-3f17b6df"
|
||||
X-Addr: www.httrack.com
|
||||
X-Fil: /robots.txt
|
||||
</pre>
|
||||
|
||||
There are also specific issues regarding this format:
|
||||
|
||||
<ul>
|
||||
<li>The data in the central directory (such as CD extra field, and CD comments) are not used</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^16 files (65535) ; in such case the total number of entries in the 32-bit central directory is 65536 (0xffff), but the presence of the 64-bit central directory is not mandatory</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory is emitted automatically (a single stored entry of 4GiB or more is not supported)</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
<b>Meta-data stored in the "extra field" of the local file headers</b><br />
|
||||
|
||||
The extra field is composed of text data, and this text data is composed of distinct lines of headers.
|
||||
The end of text, <b>or</b> a double CR/LF, mark the end of this zone.
|
||||
This method allows you to optionally store original HTTP headers just after the "meta-data" headers for informational use.<br />
|
||||
|
||||
<br />
|
||||
<b>The status line (the first headers line)</b><br />
|
||||
|
||||
Status-Line = HTTP-Version SP Status-Code SP X-Reason-Phrase CRLF<br />
|
||||
|
||||
<br />
|
||||
<b>Other lines:</b><br />
|
||||
|
||||
<br />
|
||||
<b>Specific fields:</b><br />
|
||||
<ul>
|
||||
<li>X-In-Cache</li><br>
|
||||
Indicates if the data are present (value=1) in the cache (that is, as ZIP data), or in an external file (value=0).
|
||||
This field MUST be the first field.
|
||||
|
||||
<li>X-StatusCode</li><br>
|
||||
The modified (by httrack) status code after processing. 304 error codes ("Not modified"), for example, are transformed into "200" codes after processing.
|
||||
|
||||
<li>X-StatusMessage</li><br>
|
||||
The modified (by httrack) status message.
|
||||
|
||||
<li>X-Size</li><br>
|
||||
The stored (either in cache, or in an external file) data size.
|
||||
|
||||
<li>X-Charset</li><br>
|
||||
The original charset.
|
||||
|
||||
<li>X-Addr</li><br>
|
||||
The original URL address part.
|
||||
|
||||
<li>X-Fil</li><br>
|
||||
The original URL path part.
|
||||
|
||||
<li>X-Save</li><br>
|
||||
The local filename, depending on user's "build structure" preferences.
|
||||
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
<b>Standard (RFC 2616) "useful" fields:</b><br />
|
||||
<ul>
|
||||
<li>Content-Type</li>
|
||||
<li>Last-Modified</li>
|
||||
<li>Etag</li>
|
||||
<li>Location</li>
|
||||
<li>Content-Disposition</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
<b>Specific fields in "BNF-like" grammar:</b><br />
|
||||
|
||||
<pre>
|
||||
X-In-Cache = "X-In-Cache" ":" 1*DIGIT
|
||||
X-StatusCode = "X-StatusCode" ":" 1*DIGIT
|
||||
X-StatusMessage = "X-StatusMessage" ":" *<TEXT, excluding CR, LF>
|
||||
X-Size = "X-Size" ":" 1*DIGIT
|
||||
X-Charset = "X-Charset" ":" value
|
||||
X-Addr = "X-Addr" ":" scheme ":" "//" authority
|
||||
X-Fil = "X-Fil" ":" rel_path
|
||||
X-Save = "X-Save" ":" rel_path
|
||||
</pre>
|
||||
|
||||
<b>RFC standard fields:</b><br />
|
||||
|
||||
<pre>
|
||||
Content-Type = "Content-Type" ":" media-type
|
||||
Last-Modified = "Last-Modified" ":" HTTP-date
|
||||
Etag = "ETag" ":" entity-tag
|
||||
Location = "Location" ":" absoluteURI
|
||||
Content-Disposition = "Content-Disposition" ":" disposition-type *( ";" disposition-parm )
|
||||
</pre>
|
||||
|
||||
<br />
|
||||
And, for your information,
|
||||
<pre>
|
||||
X-Reason-Phrase = *<TEXT, with a maximum of 32 characters, and excluding CR, LF>
|
||||
</pre>
|
||||
|
||||
|
||||
<a name="orig">Note: Because the URLs may have an unexpected format, especially with double "/" inside, and other reserved characters ("?", "&" ..),
|
||||
various ZIP uncompressors can potentially have troubles accessing or decompressing the data.
|
||||
Libraries should generally handle this peculiar format, however.
|
||||
</a>
|
||||
|
||||
<br /><br />
|
||||
|
||||
<!-- doc-chrome:bottom -->
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
|
||||
|
||||
<footer>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</footer>
|
||||
<!-- /doc-chrome:bottom -->
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
|
||||
|
||||
217
html/changes.html
Normal file
217
html/changes.html
Normal file
@@ -0,0 +1,217 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<!-- doc-chrome:head -->
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="The change report HTTrack writes after an update, field by field.">
|
||||
<link rel="stylesheet" href="doc.css">
|
||||
<script src="doc.js" defer></script>
|
||||
<!-- /doc-chrome:head -->
|
||||
<title>HTTrack change report format</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- doc-chrome:top -->
|
||||
<a class="skip" href="#main">Skip to content</a>
|
||||
|
||||
<header class="masthead">
|
||||
<img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier">
|
||||
<div class="tagline">Open Source offline browser</div>
|
||||
</header>
|
||||
|
||||
<div class="wrap">
|
||||
|
||||
<nav class="toc" aria-label="Documentation">
|
||||
<h2>On this page</h2>
|
||||
<ul>
|
||||
<li><a href="#what-changed-means">What "changed" means</a></li>
|
||||
<li><a href="#with-the-cache-off">With the cache off</a></li>
|
||||
<li><a href="#fields">Fields</a></li>
|
||||
<li><a href="#encoding">Encoding</a></li>
|
||||
<li><a href="#example">Example</a></li>
|
||||
<li><a href="#notes">Notes</a></li>
|
||||
</ul>
|
||||
<h2>Start here</h2>
|
||||
<ul>
|
||||
<li><a href="guide.html">Interface guide</a></li>
|
||||
<li><a href="faq.html">FAQ and troubleshooting</a></li>
|
||||
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
|
||||
</ul>
|
||||
<h2>Command line</h2>
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line guide</a></li>
|
||||
<li><a href="httrack.man.html">Option reference</a></li>
|
||||
<li><a href="filters.html">Filter syntax</a></li>
|
||||
</ul>
|
||||
<h2>Developers</h2>
|
||||
<ul>
|
||||
<li><a href="dev.html">Programming</a></li>
|
||||
<li><a href="library.html">libhttrack API</a></li>
|
||||
<li><a href="plug.html">Callbacks</a></li>
|
||||
<li><a href="scripting.html">Scripting</a></li>
|
||||
<li><a href="cache.html">Cache format</a></li>
|
||||
<li><a href="changes.html" aria-current="page" class="here">Change report format</a></li>
|
||||
</ul>
|
||||
<h2>More</h2>
|
||||
<ul>
|
||||
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
|
||||
<li><a href="contact.html">Contact and credits</a></li>
|
||||
<li><a href="index.html">All documentation</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main id="main">
|
||||
<!-- /doc-chrome:top -->
|
||||
|
||||
<h1>Change report format specification</h1>
|
||||
|
||||
<br />
|
||||
|
||||
Run with <tt>--changes</tt> (<tt>-%d</tt>), HTTrack writes <tt>hts-changes.json</tt>
|
||||
in the project directory, next to <tt>hts-log.txt</tt>, describing what the crawl
|
||||
left new, changed, unchanged and gone compared to the previous mirror. The file is
|
||||
rewritten from scratch at the end of every run, and the log carries a one-line
|
||||
summary of the same counts.
|
||||
|
||||
<br /><br />
|
||||
|
||||
<h3 id="what-changed-means">What "changed" means</h3>
|
||||
|
||||
A resource is changed when its bytes differ, not when the server merely re-sent
|
||||
it. HTTrack compares the payload it just received against the copy the previous
|
||||
run left behind: for pages it parses, the previous payload comes from the cache
|
||||
(the file on disk carries the mirror footer and its crawl date, so its bytes
|
||||
differ on every run); for everything else, the mirrored file is the payload
|
||||
verbatim and is compared directly.
|
||||
|
||||
<br /><br />
|
||||
|
||||
Where no digest can be taken on either side, because the cache is disabled or
|
||||
the previous copy is gone, the report falls back to the transfer signal, and a
|
||||
server that answers 200 rather than 304 reads as changed. Keeping the cache on
|
||||
(the default) is what makes the report precise.
|
||||
|
||||
<br /><br />
|
||||
|
||||
<h3 id="with-the-cache-off">With the cache off</h3>
|
||||
|
||||
<tt>--cache=0</tt> costs the report more than the digest of a parsed page. The
|
||||
mirror's file index (<tt>hts-cache/new.lst</tt>) is what records which files a
|
||||
run produced, so without it there is no previous mirror to subtract from: nothing
|
||||
is reported <tt>gone</tt>, and whether the run is a first crawl cannot be decided
|
||||
at all, which <tt>first_crawl</tt> states as <tt>null</tt> rather than guess. What
|
||||
is on disk is still compared byte for byte, so the other three lists stay
|
||||
meaningful, except for the pages HTTrack parses: those have no cached payload to
|
||||
compare against and fall back to the transfer signal.
|
||||
|
||||
<br /><br />
|
||||
|
||||
<h3 id="fields">Fields</h3>
|
||||
|
||||
<ul>
|
||||
<li><tt>schema</tt>: format version, currently <tt>1</tt>. It is bumped only
|
||||
on an incompatible change; new fields may appear without one.</li>
|
||||
<li><tt>generator</tt>: the HTTrack build that wrote the file.</li>
|
||||
<li><tt>date</tt>: when the report was written, UTC, <tt>YYYY-MM-DDThh:mm:ssZ</tt>.</li>
|
||||
<li><tt>first_crawl</tt>: true when no index of a previous mirror
|
||||
(<tt>hts-cache/old.lst</tt>) was found, so there was nothing to compare against and
|
||||
everything is listed as new. Null when the run kept no index at all and the
|
||||
question cannot be answered (see above).</li>
|
||||
<li><tt>partial</tt>: true when the report ran out of memory and lists only
|
||||
part of the mirror.</li>
|
||||
<li><tt>purged</tt>: true when <tt>--purge-old</tt> was in effect, so the
|
||||
files under <tt>gone</tt> were also deleted from disk.</li>
|
||||
<li><tt>counts</tt>: the size of each of the four lists.</li>
|
||||
<li><tt>new</tt>, <tt>changed</tt>, <tt>unchanged</tt>, <tt>gone</tt>: the
|
||||
lists themselves. Every mirrored file appears in exactly one of them.</li>
|
||||
</ul>
|
||||
|
||||
Each entry is an object:
|
||||
|
||||
<ul>
|
||||
<li><tt>url</tt>: the absolute URL the file came from. Empty under
|
||||
<tt>gone</tt>: deletions are computed from the mirror's file index, which records
|
||||
paths, not URLs.</li>
|
||||
<li><tt>file</tt>: the path relative to the mirror root, with forward
|
||||
slashes. This is the entry's identity: a URL and a redirect that resolve to the
|
||||
same local file are one entry, not two.</li>
|
||||
<li><tt>size</tt>: the mirrored file's size in bytes, absent when the file
|
||||
is not on disk.</li>
|
||||
<li><tt>previous_size</tt>: under <tt>changed</tt> only, the size of the
|
||||
copy the previous run left.</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
|
||||
<h3 id="encoding">Encoding</h3>
|
||||
|
||||
The file is JSON, UTF-8. URLs and local paths reach HTTrack as raw bytes and are
|
||||
not guaranteed to be valid UTF-8; any byte sequence that is not becomes
|
||||
U+FFFD (<tt>\ufffd</tt>), so the file always parses. Compare on <tt>file</tt>
|
||||
rather than on <tt>url</tt> when a mirror is known to carry legacy-charset URLs.
|
||||
|
||||
<br /><br />
|
||||
|
||||
<h3 id="example">Example</h3>
|
||||
|
||||
<pre>
|
||||
{
|
||||
"schema": 1,
|
||||
"generator": "HTTrack Website Copier/3.49-14",
|
||||
"date": "2026-07-26T15:29:03Z",
|
||||
"first_crawl": false,
|
||||
"partial": false,
|
||||
"purged": true,
|
||||
"counts": { "new": 1, "changed": 1, "unchanged": 1, "gone": 1 },
|
||||
"new": [
|
||||
{ "url": "http://example.com/d.html", "file": "example.com/d.html", "size": 280 }
|
||||
],
|
||||
"changed": [
|
||||
{ "url": "http://example.com/a.html", "file": "example.com/a.html", "size": 281, "previous_size": 273 }
|
||||
],
|
||||
"unchanged": [
|
||||
{ "url": "http://example.com/b.html", "file": "example.com/b.html", "size": 277 }
|
||||
],
|
||||
"gone": [
|
||||
{ "url": "", "file": "example.com/c.html" }
|
||||
]
|
||||
}
|
||||
</pre>
|
||||
|
||||
<br /><br />
|
||||
|
||||
<h3 id="notes">Notes</h3>
|
||||
|
||||
<ul>
|
||||
<li>A file listed under <tt>gone</tt> is only deleted when <tt>--purge-old</tt> is
|
||||
on. Left in place it drops out of the mirror's index, so it is reported once and
|
||||
not again.</li>
|
||||
<li>A resource whose local file name changed since the previous mirror (a new
|
||||
MIME type, say) is reported as <tt>new</tt> under its new name; the old name is
|
||||
reported as <tt>gone</tt> only if the file is still on disk. The two entries are
|
||||
not paired.</li>
|
||||
<li>A resource this run tried and failed to transfer also drops out of the
|
||||
mirror's index, but its previous copy is untouched, so it is reported
|
||||
<tt>unchanged</tt>. Under <tt>--purge-old</tt> that copy is deleted anyway, and
|
||||
the report says <tt>gone</tt> to match.</li>
|
||||
<li>A run that transfers no data at all is rolled back: HTTrack restores the
|
||||
previous cache generation and leaves the previous report in place, so a lost
|
||||
connection does not overwrite a good report with an empty one.</li>
|
||||
<li>Content diffs, and keeping the previous copy of a changed page, are out of
|
||||
scope: both change what a mirror directory contains.</li>
|
||||
</ul>
|
||||
|
||||
<br /><br />
|
||||
|
||||
<!-- doc-chrome:bottom -->
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
|
||||
|
||||
<footer>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</footer>
|
||||
<!-- /doc-chrome:bottom -->
|
||||
|
||||
</body>
|
||||
</html>
|
||||
159
html/cmddoc.html
159
html/cmddoc.html
@@ -1,155 +1,10 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<title>Moved to the command-line guide</title>
|
||||
<link rel="canonical" href="cmdguide.html">
|
||||
<meta http-equiv="refresh" content="0; url=cmdguide.html">
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>Command-Line Documentation</em></h2>
|
||||
|
||||
<br>
|
||||
|
||||
The command-line version
|
||||
|
||||
<ul>
|
||||
<li><a href="options.html">Command line Options</a></li>
|
||||
<br>List of all powerful command line options<br><br>
|
||||
<li>How to use httrack command-line version:</li>
|
||||
<ul>
|
||||
<li>Open a shell window</li>
|
||||
<br>
|
||||
<br>
|
||||
<li>Type in <tt>httrack</tt> (or the complete path to the httrack executable)</li>
|
||||
<br><small><tt>httrack</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Add the URLs, separated by a blank space</li>
|
||||
<br><small><tt>httrack www.example.com/foo/</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>If you need, add some options (see the <a href="options.html">option list</a>)</li>
|
||||
<br><small><tt>httrack www.example.com/foo/ -O "/webs" -N4 -P proxy.myhost.com:3128</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Launch the command line, and wait until the mirror is finishing</li>
|
||||
<br><small>You can (especially on the Unix release) press ^C to stop the mirror or put httrack in background</small>
|
||||
<br>
|
||||
<br>
|
||||
</ul>
|
||||
</ul>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
<body><p>This page is now part of <a href="cmdguide.html">the HTTrack command-line guide</a>.</p></body>
|
||||
</html>
|
||||
|
||||
|
||||
511
html/cmdguide.html
Normal file
511
html/cmdguide.html
Normal file
@@ -0,0 +1,511 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<!-- doc-chrome:head -->
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="A task-oriented guide to the httrack command line: scope, filters, limits, logins, proxies, updates and recipes.">
|
||||
<link rel="stylesheet" href="doc.css">
|
||||
<script src="doc.js" defer></script>
|
||||
<!-- /doc-chrome:head -->
|
||||
<title>HTTrack command-line guide</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- doc-chrome:top -->
|
||||
<a class="skip" href="#main">Skip to content</a>
|
||||
|
||||
<header class="masthead">
|
||||
<img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier">
|
||||
<div class="tagline">Open Source offline browser</div>
|
||||
</header>
|
||||
|
||||
<div class="wrap">
|
||||
|
||||
<nav class="toc" aria-label="Documentation">
|
||||
<h2>On this page</h2>
|
||||
<ul>
|
||||
<li><a href="#quickstart">1. Quick start</a></li>
|
||||
<li><a href="#scope">2. Scope: how far the crawl reaches</a></li>
|
||||
<li><a href="#filters">3. Filters and scan rules</a></li>
|
||||
<li><a href="#limits">4. Limits and politeness</a></li>
|
||||
<li><a href="#names">5. File names and types</a></li>
|
||||
<li><a href="#links">6. Links and page building</a></li>
|
||||
<li><a href="#identity">7. Identity, cookies and login</a></li>
|
||||
<li><a href="#proxy">8. Proxy and network</a></li>
|
||||
<li><a href="#update">9. Update and cache</a></li>
|
||||
<li><a href="#experts">10. Experts and scripting</a></li>
|
||||
<li><a href="#recipes">11. Recipes</a></li>
|
||||
</ul>
|
||||
<h2>Start here</h2>
|
||||
<ul>
|
||||
<li><a href="guide.html">Interface guide</a></li>
|
||||
<li><a href="faq.html">FAQ and troubleshooting</a></li>
|
||||
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
|
||||
</ul>
|
||||
<h2>Command line</h2>
|
||||
<ul>
|
||||
<li><a href="cmdguide.html" aria-current="page" class="here">Command-line guide</a></li>
|
||||
<li><a href="httrack.man.html">Option reference</a></li>
|
||||
<li><a href="filters.html">Filter syntax</a></li>
|
||||
</ul>
|
||||
<h2>Developers</h2>
|
||||
<ul>
|
||||
<li><a href="dev.html">Programming</a></li>
|
||||
<li><a href="library.html">libhttrack API</a></li>
|
||||
<li><a href="plug.html">Callbacks</a></li>
|
||||
<li><a href="scripting.html">Scripting</a></li>
|
||||
<li><a href="cache.html">Cache format</a></li>
|
||||
<li><a href="changes.html">Change report format</a></li>
|
||||
</ul>
|
||||
<h2>More</h2>
|
||||
<ul>
|
||||
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
|
||||
<li><a href="contact.html">Contact and credits</a></li>
|
||||
<li><a href="index.html">All documentation</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main id="main">
|
||||
<!-- /doc-chrome:top -->
|
||||
|
||||
<h1>Command-Line Guide</h1>
|
||||
|
||||
<p>This is a task-oriented guide to the <tt>httrack</tt> command line: how to do the
|
||||
things people actually ask for, and the handful of defaults that surprise
|
||||
newcomers. It sits on top of the
|
||||
<a href="httrack.man.html">generated manual page</a>, which lists every option in
|
||||
full. When you want the exhaustive detail for a flag, that page is the reference;
|
||||
this one is the map.</p>
|
||||
|
||||
<p>Two habits before anything else. First, HTTrack has its own options: they are not
|
||||
wget or curl flags, so reach for the tables here rather than guessing. Second,
|
||||
when a mirror does something you did not expect, the answer is almost always in
|
||||
the log. Every project writes <tt>hts-log.txt</tt> (and <tt>hts-err.txt</tt>) into
|
||||
its output directory, and those files name every URL that was refused, redirected,
|
||||
or filtered out. Read them first.</p>
|
||||
|
||||
<h3 id="quickstart">1. Quick start</h3>
|
||||
|
||||
<p>A mirror is one command: a start URL and an output directory.</p>
|
||||
|
||||
<p><tt>httrack https://example.com/ --path mydir</tt></p>
|
||||
|
||||
<p>With no other options HTTrack mirrors that site, stays on the same host, follows
|
||||
links to any depth, rebuilds them to browse offline, and stores everything under
|
||||
<tt>mydir</tt>. The same directory also holds the log files and the
|
||||
<tt>hts-cache/</tt> folder that makes a later update or resume possible.</p>
|
||||
|
||||
<p>Two defaults are worth knowing up front, because both catch people out:</p>
|
||||
<ul>
|
||||
<li>HTTrack throttles itself to about <b>100 KB/s</b> even when you pass no rate
|
||||
option. If a mirror feels slow, that is why. See
|
||||
<a href="#limits">Limits</a> for how to lift it.</li>
|
||||
<li>The download proceeds as a well-behaved robot: it identifies itself as
|
||||
<tt>HTTrack</tt>, obeys <tt>robots.txt</tt>, and sends a Referer with each
|
||||
request. A site that blocks that behavior needs the levers in
|
||||
<a href="#identity">Identity</a>, not brute force.</li>
|
||||
</ul>
|
||||
|
||||
<h3 id="scope">2. Scope: how far the crawl reaches</h3>
|
||||
|
||||
<p>Scope decides which links HTTrack is even willing to follow, before any filter
|
||||
you write. Get this right and most "it downloaded too much" or "it only grabbed
|
||||
the index" problems disappear.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--depth (-r)</tt></td><td>Maximum link depth. <b>The start page is level 1</b>, so one level of links out is <tt>-r2</tt>, not <tt>-r1</tt>.</td></tr>
|
||||
<tr><td><tt>--stay-on-same-address (-a), --stay-on-same-domain (-d), --stay-on-same-tld (-l), --go-everywhere (-e)</tt></td><td>How far off the starting host the crawl may travel: same address (host), same principal domain, same top-level domain (for example .com), or everywhere. The default keeps you on the starting host.</td></tr>
|
||||
<tr><td><tt>--can-go-down (-D), --can-go-up (-U), --stay-on-same-dir (-S), --can-go-up-and-down (-B)</tt></td><td>Directory travel: down into subdirectories only, up to parent directories only, stay in the same directory, or both up and down.</td></tr>
|
||||
<tr><td><tt>--near (-n)</tt></td><td>Also fetch non-HTML files "near" a followed link, such as an image linked from a page you kept but hosted elsewhere.</td></tr>
|
||||
<tr><td><tt>--ext-depth (-%e)</tt></td><td>How many levels of external links to follow once the crawl leaves your scope (default 0).</td></tr>
|
||||
<tr><td><tt>--test (-t)</tt></td><td>Also HEAD-test links that fall outside the scope, which are normally refused, without downloading them: a way to see what scope is excluding.</td></tr>
|
||||
<tr><td><tt>--sitemap (-%m), --sitemap-url URL (-%mu)</tt></td><td>Also take start URLs from the site's sitemap, for pages nothing links to. Off by default.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>Link-following only finds what something links to. Anything a site publishes
|
||||
solely in its sitemap is invisible to HTTrack unless you ask for it.
|
||||
<tt>--sitemap</tt> reads the start host's <tt>robots.txt</tt> for
|
||||
<tt>Sitemap:</tt> lines and falls back to <tt>/sitemap.xml</tt>;
|
||||
<tt>--sitemap-url</tt> names one directly. Nested <tt>sitemapindex</tt> files
|
||||
and gzipped <tt>.xml.gz</tt> sitemaps are followed. The URLs found become start
|
||||
URLs with the full depth budget, but they still go through your filters and
|
||||
scope rules, so a sitemap cannot widen a crawl you deliberately narrowed. It is
|
||||
off by default because a sitemap can list thousands of pages nothing links
|
||||
to.</p>
|
||||
|
||||
<p>One surprise worth knowing: a sitemap you name with <tt>--sitemap-url</tt>,
|
||||
and one the site itself declares in <tt>robots.txt</tt>, are fetched even when
|
||||
<tt>robots.txt</tt> disallows that path, because naming or declaring a sitemap
|
||||
is an invitation to read it. Only the guessed <tt>/sitemap.xml</tt> obeys a
|
||||
<tt>Disallow</tt>. The URLs listed inside are gated normally either way.</p>
|
||||
|
||||
<p>The single most common surprise is "only the home page came down." That is
|
||||
usually not a scope option at all: it is an off-host redirect. A start URL of
|
||||
<tt>http://example.com/</tt> that redirects to <tt>https://www.example.com/</tt>
|
||||
lands you on a different host, and same-host scope stops the crawl there. Start
|
||||
from the final URL, or add a filter that re-admits the real host (see
|
||||
<a href="#filters">Filters</a>). The log will show the redirect.</p>
|
||||
|
||||
<p><tt>-n</tt> is the fix for pages that render locally without their images or
|
||||
stylesheets: it lets HTTrack pull in requisites that sit just outside scope. Note
|
||||
that its embedded-asset handling (following <tt>img</tt>, <tt>link</tt>,
|
||||
<tt>script</tt>, <tt>style</tt> and HTML5 <tt>source</tt>/<tt>track</tt> targets
|
||||
past the normal depth and filter limits) applies only when <tt>-n</tt> is on; it
|
||||
is not automatic. It can also over-fetch by dragging in a whole external host from
|
||||
a single link, in which case name the assets you want with a filter instead.</p>
|
||||
|
||||
<h3 id="filters">3. Filters and scan rules</h3>
|
||||
|
||||
<p>Filters are the number-one source of confusion, and also the tool that solves
|
||||
most scope problems once you understand them. A filter is a rule that accepts
|
||||
(<tt>+</tt>) or rejects (<tt>-</tt>) URLs by pattern. The sign is mandatory:
|
||||
<tt>+pattern</tt> adds, <tt>-pattern</tt> removes, and a bare pattern is an error.</p>
|
||||
|
||||
<p>The rules that matter:</p>
|
||||
<ul>
|
||||
<li><b>Last match wins.</b> Rules are applied in order and the last one that
|
||||
matches a URL decides its fate. Order your rules from general to specific.</li>
|
||||
<li><b>Wildcards.</b> <tt>*</tt> matches any run of characters;
|
||||
<tt>*[a-z]</tt>, <tt>*[0-9]</tt> and similar classes match sets. So
|
||||
<tt>+*.pdf</tt> means "any URL ending in .pdf".</li>
|
||||
<li><b>Whitelisting.</b> To keep one site and nothing else, deny everything then
|
||||
re-admit the host: <tt>"-*" "+example.com/*"</tt>. A lone <tt>+</tt> rule only
|
||||
adds to the default scope; it never restricts.</li>
|
||||
<li><b>Size rules.</b> <tt>*[>100000]</tt> and <tt>*[<1000]</tt> filter by
|
||||
byte size. Because size is only known once the transfer starts, an oversize file
|
||||
is fetched partway and then aborted, not skipped for free.</li>
|
||||
<li><b>mime: rules.</b> A rule like <tt>-mime:video/*</tt> matches the
|
||||
<tt>Content-Type</tt>. That type is only known after the response headers arrive,
|
||||
so a mime rule <b>cannot stop a request</b>; it can only abort the body. Use a
|
||||
URL pattern when you want to avoid the fetch entirely.</li>
|
||||
</ul>
|
||||
|
||||
<p>Quote your filters. Shells treat <tt>*</tt>, <tt>[</tt> and sometimes <tt>+</tt>
|
||||
specially, so wrap each rule in quotes as shown above. The full pattern language,
|
||||
with tables for wildcards, size and mime, is in
|
||||
<a href="filters.html">the filters page</a>, and the
|
||||
<a href="faq.html">FAQ</a> has a worked tutorial.</p>
|
||||
|
||||
<p><b>robots.txt.</b> By default HTTrack obeys <tt>robots.txt</tt> (<tt>-s2</tt>).
|
||||
<tt>-s0</tt> ignores it entirely, <tt>-s1</tt> obeys it but lets one of your
|
||||
<tt>+</tt> filters override a disallow for a URL you explicitly asked for. Note
|
||||
that a <tt>403 Forbidden</tt> is a server refusal, not a robots rule: robots
|
||||
options will not help there. That is an
|
||||
<a href="#identity">identity</a> problem.</p>
|
||||
|
||||
<h4>Filter wildcards</h4>
|
||||
<p>Inside a filter pattern, <tt>*</tt> matches any run of characters; a few
|
||||
bracket forms match narrower sets. The full table, with size and mime rules, is on
|
||||
<a href="filters.html">the filters page</a>.</p>
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Wildcard</b></td><td><b>Matches</b></td><td><b>Example</b></td></tr>
|
||||
<tr><td><tt>*</tt></td><td>any run of characters</td><td><tt>+*.pdf</tt> — any URL ending <tt>.pdf</tt></td></tr>
|
||||
<tr><td><tt>*[file]</tt>, <tt>*[name]</tt></td><td>one path segment (any char but <tt>/</tt> and <tt>?</tt>)</td><td><tt>example.com/*[file]/</tt> — a directory-index page</td></tr>
|
||||
<tr><td><tt>*[path]</tt></td><td>a path, slashes allowed (any char but <tt>?</tt>)</td><td><tt>example.com/*[path].zip</tt></td></tr>
|
||||
<tr><td><tt>*[param]</tt></td><td>an optional query string</td><td><tt>page.html*[param]</tt> matches with or without <tt>?...</tt></td></tr>
|
||||
<tr><td><tt>*[a,b,c]</tt></td><td>any one character in the set</td><td><tt>*[a,b,c].txt</tt></td></tr>
|
||||
<tr><td><tt>*[a-z]</tt></td><td>any one character in the range</td><td><tt>img*[0-9].gif</tt></td></tr>
|
||||
<tr><td><tt>*[\x]</tt></td><td>the literal character x (escapes <tt>* [ ] \</tt>)</td><td><tt>*[\*]</tt> matches a real <tt>*</tt></td></tr>
|
||||
<tr><td><tt>*[<NN]</tt>, <tt>*[>NN]</tt></td><td>file size in KB below / above NN</td><td><tt>-*.gif*[<5]</tt> skips GIFs under 5 KB</td></tr>
|
||||
<tr><td><tt>*[]</tt></td><td>end anchor: nothing may follow</td><td><tt>*.html*[]</tt> rejects <tt>i.html?p=1</tt></td></tr>
|
||||
</table>
|
||||
|
||||
<h3 id="limits">4. Limits and politeness</h3>
|
||||
|
||||
<p>HTTrack ships cautious on purpose: it is easy to hammer a small site by
|
||||
accident, and the <a href="abuse.html">abuse page</a> is worth a read. The limits
|
||||
below let you go faster when you own the target, and slower when you do not.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--max-rate (-A)</tt></td><td>Maximum transfer rate in bytes/sec. <b>The default is about 100 KB/s even without this flag.</b> Raise it to go faster.</td></tr>
|
||||
<tr><td><tt>--sockets (-c)</tt></td><td>Number of parallel connections (default 4). <tt>--tiny</tt>, <tt>--wide</tt> and <tt>--ultrawide</tt> are presets.</td></tr>
|
||||
<tr><td><tt>--connection-per-second (-%c)</tt></td><td>New connections opened per second (default 5).</td></tr>
|
||||
<tr><td><tt>--max-size (-M)</tt></td><td>Stop after N bytes <b>received from the network</b> across the whole mirror (this counts what was transferred, not what was saved).</td></tr>
|
||||
<tr><td><tt>--max-time (-E)</tt></td><td>Stop after N seconds of wall-clock time.</td></tr>
|
||||
<tr><td><tt>--max-files (-m)</tt></td><td>Per-file size caps.</td></tr>
|
||||
<tr><td><tt>--timeout (-T), --retries (-R), --min-rate (-J), --host-control (-H)</tt></td><td>Idle timeout, retry count, minimum acceptable rate, and host-ban behavior for slow or dead hosts.</td></tr>
|
||||
<tr><td><tt>--max-pause (-G), --pause (-%G)</tt></td><td>Pause the mirror at N bytes, or pause between files, to spread the load.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>The security clamps.</b> To keep an accidental typo from turning into a flood,
|
||||
HTTrack silently caps a few values: at most 8 connections (<tt>-c</tt>), at most
|
||||
10 MB/s (<tt>-A</tt>), and at most 5 new connections per second (<tt>-%c</tt>).
|
||||
Ask for more and you get the ceiling, quietly. The single flag
|
||||
<tt>--disable-security-limits</tt> lifts all three (the short form <tt>-%!</tt>
|
||||
also works, but the bare <tt>!</tt> is awkward to type safely in a shell). Use it
|
||||
only against infrastructure you are allowed to load that hard.</p>
|
||||
|
||||
<h3 id="names">5. File names and types</h3>
|
||||
|
||||
<p>Where local files land, and what they are called, is controlled by the naming
|
||||
options. This is the second-biggest source of "why did it do that" questions,
|
||||
usually about a URL like <tt>/article?id=42</tt> or a <tt>.php</tt> page that is
|
||||
really HTML.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--structure (-N)</tt></td><td>The local path and name layout. Presets are numeric, and you can also give a template such as <tt>--structure "%h%p/%n%q.%t"</tt>.</td></tr>
|
||||
<tr><td><tt>--long-names (-L)</tt></td><td>Long names, 8.3 names, or ISO9660 for CD masters.</td></tr>
|
||||
<tr><td><tt>--assume (-%A)</tt></td><td>Assume a MIME type for an extension, for example <tt>--assume php=text/html</tt>. This also skips the extra HEAD probe HTTrack would otherwise send to learn the type.</td></tr>
|
||||
<tr><td><tt>--delayed-type-check (-%N), --cached-delayed-type-check (-%D), --check-type (-u), -%t</tt></td><td>When and how the content type is checked, and whether the original extension is kept.</td></tr>
|
||||
<tr><td><tt>--include-query-string (-%q), --strip-query (-%g)</tt></td><td>Whether the query string appears in the local filename, and whether query keys are stripped when deciding if two URLs are the same file.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>The <tt>-N</tt> presets are built from modular arithmetic on the name fields, so
|
||||
undocumented number combinations often "work" by accident. If you care about the
|
||||
exact layout, use an explicit template (the <tt>%h %p %n %q %t</tt> placeholders)
|
||||
rather than a magic number, and check the result on a small crawl first.</p>
|
||||
|
||||
<p>A dynamic page served as <tt>.php</tt> or <tt>.asp</tt> that is actually HTML is
|
||||
the classic case: without help it can be saved with an extension a browser will
|
||||
not open locally. <tt>--assume php=text/html</tt> fixes both the extension and the
|
||||
naming.</p>
|
||||
|
||||
<h3 id="links">6. Links and page building</h3>
|
||||
|
||||
<p>After a page is downloaded, HTTrack parses it for more links and rewrites the
|
||||
ones it kept so the local copy browses offline. These options tune both halves.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--keep-links (-K)</tt></td><td>How links are rewritten in saved pages. The numbering is inverted from what you might guess: bare <tt>-K</tt> keeps <b>absolute</b> URLs, and <tt>-K0</tt> is the <b>relative</b> default. <tt>-K3</tt> keeps absolute URIs, <tt>-K4</tt> keeps the original links.</td></tr>
|
||||
<tr><td><tt>--replace-external (-x), --generate-errors (-o)</tt></td><td>Replace external links with an error page, and generate an error page for links that failed.</td></tr>
|
||||
<tr><td><tt>--preserve (-%p), --disable-passwords (-%x)</tt></td><td>Leave HTML untouched (no rewriting), and strip passwords out of saved links.</td></tr>
|
||||
<tr><td><tt>--extended-parsing (-%P), --parse-java (-j)</tt></td><td>Aggressive link discovery, and how much script content is parsed for links.</td></tr>
|
||||
<tr><td><tt>--mime-html (-%M)</tt></td><td>Save the whole mirror as a single MIME-encapsulated <tt>.mht</tt> archive (<tt>index.mht</tt>).</td></tr>
|
||||
<tr><td><tt>--single-file (-%Z), --single-file-max-size N</tt></td><td>Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded as <tt>data:</tt> URIs. Assets over the cap (10 MB by default) keep their link, as do audio, video, and the links from one page to another. A sibling of <tt>-%M</tt>, not a replacement: see the recipe below for which to pick.</td></tr>
|
||||
<tr><td><tt>--index (-I), --build-top-index (-%i), --search-index (-%I)</tt></td><td>Build a per-mirror index, a top index across projects, and a searchable keyword index.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>HTTrack finds links by parsing HTML and CSS. It does not run JavaScript, so any
|
||||
URL a page builds at runtime in script (a lazy-loaded image, a
|
||||
JavaScript-assembled path) is invisible to the crawler and will be missing from
|
||||
the mirror. There is no flag that fixes this; the asset has to appear in the
|
||||
static HTML or CSS to be found. <tt>-%P</tt> widens discovery for links that are
|
||||
present but awkwardly formatted, not for links that do not exist until script
|
||||
runs.</p>
|
||||
|
||||
<h3 id="identity">7. Identity, cookies and login</h3>
|
||||
|
||||
<p>By default HTTrack is an honest robot: it sends a <tt>User-Agent</tt> of
|
||||
<tt>HTTrack</tt>, a Referer with each link (which reveals the crawl path to the
|
||||
server), and obeys robots. Plenty of sites filter exactly that profile. These
|
||||
options control what HTTrack says about itself.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--user-agent (-F)</tt></td><td>The <tt>User-Agent</tt>. Set a browser string to get past crawler blocks; <tt>--user-agent ""</tt> sends none.</td></tr>
|
||||
<tr><td><tt>--referer (-%R), --from (-%E), --language (-%l), --accept (-%a)</tt></td><td>Referer, From, Accept-Language and Accept headers.</td></tr>
|
||||
<tr><td><tt>--headers (-%X)</tt></td><td>Add raw header lines to every request.</td></tr>
|
||||
<tr><td><tt>--footer (-%F)</tt></td><td>A footer written into saved pages (on disk, not a network header). See <b>Footer fields</b> below.</td></tr>
|
||||
<tr><td><tt>--cookies (-b), --cookies-file (-%K)</tt></td><td>Accept cookies, and preload a Netscape <tt>cookies.txt</tt>.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>Footer fields.</b> A footer with no <tt>%s</tt> may reference named fields:
|
||||
<tt>{addr}</tt>, <tt>{path}</tt>, <tt>{url}</tt>, <tt>{date}</tt> (mirror time),
|
||||
<tt>{lastmodified}</tt> (the page's Last-Modified), <tt>{version}</tt>,
|
||||
<tt>{mime}</tt>, <tt>{charset}</tt>, <tt>{status}</tt> and <tt>{size}</tt>; write
|
||||
<tt>{{</tt> or <tt>}}</tt> for a literal brace. A footer that contains <tt>%s</tt>
|
||||
keeps the older positional form (host, path, date in that order). Example:
|
||||
<tt>-%F "<!-- Mirrored from {url} on {date} -->"</tt>.</p>
|
||||
|
||||
<p><b>Login.</b> For HTTP Basic auth, put the credentials in the URL:
|
||||
<tt>http://user:pass@host/</tt>. An <tt>@</tt> inside the username must be written
|
||||
<tt>%40</tt>. Only Basic is supported, not Digest.</p>
|
||||
|
||||
<p>For cookie or form logins, the simplest path is to log in with a browser, export
|
||||
its <tt>cookies.txt</tt>, and drop that file in the project directory so HTTrack
|
||||
sends the session cookie. For a form that needs a POST, <tt>--catchurl</tt> can
|
||||
capture the exact request your browser sends and replay it. A few cookie caveats
|
||||
to know: expiry is ignored, there is a silent cap of about 8 cookies sent per
|
||||
request, and <tt>-b0</tt> disables cookies and the reuse of Basic credentials
|
||||
across links at the same time.</p>
|
||||
|
||||
<h3 id="proxy">8. Proxy and network</h3>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--proxy (-P)</tt></td><td>Route through a proxy. HTTP, SOCKS5 and CONNECT are supported: <tt>-P host:8080</tt>, <tt>-P socks5://host:1080</tt>, <tt>-P connect://host:443</tt>, with optional <tt>user:pass@</tt>.</td></tr>
|
||||
<tr><td><tt>--httpproxy-ftp (-%f)</tt></td><td>Send FTP requests through the HTTP proxy.</td></tr>
|
||||
<tr><td><tt>--protocol (-@i)</tt></td><td>Prefer IPv4 or IPv6.</td></tr>
|
||||
<tr><td><tt>--http-10 (-%h), --keep-alive (-%k), --disable-compression (-%z)</tt></td><td>Force HTTP/1.0 (drops keep-alive and compression, useful for fragile CGI), toggle keep-alive, and toggle compression.</td></tr>
|
||||
<tr><td><tt>--bind (-%b), --tolerant (-%B)</tt></td><td>Bind to a local address, and accept technically-bogus responses some servers send.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>Two network facts worth stating plainly. Over SOCKS5, HTTrack always resolves
|
||||
host names at the proxy (remote DNS) for both <tt>socks5://</tt> and
|
||||
<tt>socks5h://</tt>, so your local resolver is never consulted. And HTTrack does
|
||||
not verify TLS certificates: HTTPS gives you an encrypted transport, but not an
|
||||
authenticated one. That is a deliberate choice for a mirroring tool, not a bug,
|
||||
but it is worth knowing if you are relying on it for trust.</p>
|
||||
|
||||
<h3 id="update">9. Update and cache</h3>
|
||||
|
||||
<p>Every project keeps a cache under <tt>hts-cache/</tt>. It records every URL that
|
||||
was fetched, together with the options you used, and it is what makes resuming and
|
||||
updating possible. It is not a size-limited scratch area you can delete: throw it
|
||||
away and you lose the ability to continue or update the mirror.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--continue</tt></td><td>Carry on an interrupted mirror, trusting the cache: it does not re-check pages already stored.</td></tr>
|
||||
<tr><td><tt>--update</tt></td><td>Re-run the mirror, revalidating each page with the server (If-Modified-Since / If-None-Match) and downloading only what changed.</td></tr>
|
||||
<tr><td><tt>--purge-old=0 (-X0)</tt></td><td>Do not purge. By default an update deletes local files that are no longer part of the mirror; <tt>--purge-old=0</tt> keeps them.</td></tr>
|
||||
<tr><td><tt>--cache (-C)</tt></td><td>Cache mode. The default already does the right thing and switches to update-checking when it detects an existing mirror.</td></tr>
|
||||
<tr><td><tt>--urlhack (-%u), --keep-www-prefix (-%j), --keep-double-slashes (-%o), --keep-query-order (-%y), --do-not-recatch (-%n), --updatehack (-%s), --store-all-in-cache (-k)</tt></td><td>URL-deduplication behavior and cache storage details.</td></tr>
|
||||
<tr><td><tt>--debug-cache (-#C), --repair-cache (-#R), --clean</tt></td><td>Inspect the cache, repair its ZIP, and erase cache plus logs.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>The purge trap.</b> An <tt>--update</tt> run rebuilds the list of files the
|
||||
mirror should contain, then deletes any previously-mirrored file that is not on the
|
||||
new list. This is what keeps a mirror in sync with a shrinking site, but it means a
|
||||
partial or interrupted update can delete files you meant to keep. If an update might
|
||||
not complete cleanly, add <tt>-X0</tt> to protect the existing tree, and expect
|
||||
dynamic pages to look "changed" on every run. See the
|
||||
<a href="cache.html">cache page</a> for the details.</p>
|
||||
|
||||
<h3 id="experts">10. Experts and scripting</h3>
|
||||
|
||||
<p>HTTrack is also a scriptable fetch-and-scan tool. These options turn off the
|
||||
mirror behavior and expose the engine.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="head"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--get URL</tt></td><td>Fetch a single file and stop. Cache, index, depth, cookies and robots are all off for this mode.</td></tr>
|
||||
<tr><td><tt>--spider --testlinks --skeleton</tt></td><td>Scan without saving, test links at depth 1, or keep HTML only. Handy for checking a site before a real crawl.</td></tr>
|
||||
<tr><td><tt>--userdef-cmd (-V)</tt></td><td>Run a shell command on each downloaded file; <tt>$0</tt> is the file path. Good for on-the-fly processing.</td></tr>
|
||||
<tr><td><tt>--callback (-%W)</tt></td><td>Load an external callback module to hook the engine.</td></tr>
|
||||
<tr><td><tt>--do-not-log (-Q), --quiet (-q), --verbose (-v), --file-log (-f), --extra-log (-z), --debug-log (-Z)</tt></td><td>Logging: quiet, no questions, verbose on screen, and the various log-to-file levels.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>The <a href="dev.html">developer page</a> covers the callback API and batch use
|
||||
in more depth.</p>
|
||||
|
||||
<h3 id="recipes">11. Recipes</h3>
|
||||
|
||||
<p>Copy-ready command lines for the tasks people ask about most. Each has the one
|
||||
gotcha that trips it up.</p>
|
||||
|
||||
<h4>Mirror one site, nothing off-host</h4>
|
||||
<p><tt>httrack https://example.com/ --path mydir</tt><br>
|
||||
<small>Same-host is already the default. The usual failure is a
|
||||
<tt>www.</tt>-to-apex or <tt>http</tt>-to-<tt>https</tt> redirect that moves you off
|
||||
host and stops the crawl; start from the final URL, or add
|
||||
<tt>"+finalhost/*"</tt>.</small></p>
|
||||
|
||||
<h4>One level of links only</h4>
|
||||
<p><tt>httrack https://example.com/ --depth=2 --path mydir</tt><br>
|
||||
<small>Depth counts the start page as level 1, so one level out is <tt>--depth=2</tt>.</small></p>
|
||||
|
||||
<h4>This site only, deny everything else</h4>
|
||||
<p><tt>httrack https://example.com/ "-*" "+example.com/*" --path mydir</tt><br>
|
||||
<small>You must deny-all first; a lone <tt>+</tt> only adds, and the last matching
|
||||
rule wins.</small></p>
|
||||
|
||||
<h4>Download the PDFs on a site</h4>
|
||||
<p><tt>httrack https://example.com/ "-*" "+https://example.com/*.html" "+https://example.com/*[path]/" "+https://example.com/*.pdf" --path mydir</tt><br>
|
||||
<small>HTTrack finds PDFs by parsing the site's HTML, so a plain
|
||||
<tt>"-*" "+example.com/*.pdf"</tt> is wrong: it prunes the pages that carry the
|
||||
links and keeps only PDFs reachable from the front page. Instead admit the HTML as
|
||||
scaffolding (<tt>*.html</tt> and <tt>*[path]/</tt> for directory-index pages at any
|
||||
depth, e.g. <tt>docs/</tt> or <tt>a/b/deep/</tt>; <tt>*[file]/</tt> would stop at one
|
||||
level), keep the PDFs, and let <tt>-*</tt> drop everything else (images,
|
||||
archives, off-site assets). PDFs on another host (a CDN or docs subdomain) are not
|
||||
included by default; allow that host too, e.g. <tt>"+docs.example.com/*.pdf"</tt>,
|
||||
or widen to <tt>"+*.pdf"</tt> for PDFs anywhere.</small></p>
|
||||
|
||||
<h4>Keep page requisites, including off-host images</h4>
|
||||
<p><tt>httrack https://example.com/blog/ --near --path mydir</tt><br>
|
||||
<small><tt>--near</tt> can over-fetch by pulling in an entire external host from one
|
||||
link; if it does, name the assets with <tt>"+cdn.example.com/*"</tt> instead.</small></p>
|
||||
|
||||
<h4>Resume an interrupted crawl</h4>
|
||||
<p><tt>httrack --continue --path mydir</tt><br>
|
||||
<small>Needs an intact <tt>hts-cache/</tt>. Deleting it loses the URL list and your
|
||||
options.</small></p>
|
||||
|
||||
<h4>Update a mirror, downloading only what changed</h4>
|
||||
<p><tt>httrack --update --path mydir</tt><br>
|
||||
<small>Purge deletes any previously-mirrored file not seen this run; add
|
||||
<tt>--purge-old=0</tt> to protect the tree against a partial update.</small></p>
|
||||
|
||||
<h4>Reach a section behind a login cookie</h4>
|
||||
<p><small>Export your browser's <tt>cookies.txt</tt> into the project directory,
|
||||
then:</small><br>
|
||||
<tt>httrack https://example.com/members/ --path mydir</tt><br>
|
||||
<small>The file must be Netscape <tt>cookies.txt</tt> in the project folder;
|
||||
<tt>--cookies=0</tt> would disable cookies and Basic-auth reuse together.</small></p>
|
||||
|
||||
<h4>Get past a crawler-UA block</h4>
|
||||
<p><tt>httrack https://example.com/ --user-agent "Mozilla/5.0 (Windows NT 10.0)" --path mydir</tt><br>
|
||||
<small>A 403 is server-side, not robots: change the User-Agent, and add
|
||||
<tt>--http-10</tt> for fragile CGI. Do not reach for <tt>--robots=0</tt>.</small></p>
|
||||
|
||||
<h4>Full-speed mirror on your own server</h4>
|
||||
<p><tt>httrack https://example.com/ --max-rate=2000000 --sockets=8 --disable-security-limits --path mydir</tt><br>
|
||||
<small>The default throttles to about 100 KB/s. <tt>--disable-security-limits</tt>
|
||||
lifts the built-in caps; use it only against infrastructure you are allowed to
|
||||
load.</small></p>
|
||||
|
||||
<h4>Save a WARC archive of the crawl</h4>
|
||||
<p><tt>httrack https://example.com/ --warc --path mydir</tt><br>
|
||||
<small>Writes a standard WARC/1.1 file (<tt>httrack-<timestamp>.warc.gz</tt>) in
|
||||
the project folder alongside the browsable mirror, not instead of it. Set the name
|
||||
with <tt>--warc-file NAME</tt> and split a large crawl with <tt>--warc-max-size N</tt>;
|
||||
add <tt>--warc-cdx</tt> for a sorted CDXJ index, or <tt>--wacz</tt> to bundle the
|
||||
archive, index and pages into one WACZ for replay tools such as
|
||||
replayweb.page.</small></p>
|
||||
|
||||
<h4>See what a re-crawl changed</h4>
|
||||
<p><tt>httrack https://example.com/ --update --changes --path mydir</tt><br>
|
||||
<small>Writes <tt>hts-changes.json</tt> in the project folder, listing every
|
||||
mirrored file as new, changed, unchanged or gone, plus a one-line summary in the
|
||||
log. "Changed" means the bytes really differ: a server that answers 200
|
||||
with the same content it served last time lands in <tt>unchanged</tt>. Deletions
|
||||
are reported whether or not <tt>--purge-old</tt> is deleting them. The format is
|
||||
documented in <a href="changes.html">the change report specification</a>.</small></p>
|
||||
|
||||
<h4>Pages you can hand to someone as one file</h4>
|
||||
<p><tt>httrack https://example.com/ --single-file --path mydir</tt><br>
|
||||
<small>Rewrites each saved page after the crawl so its stylesheets, scripts,
|
||||
images and fonts are embedded as <tt>data:</tt> URIs. The mirror stays a normal
|
||||
browsable tree, with links between pages relative and the assets still on disk,
|
||||
but any single <tt>.html</tt> file can now be mailed or archived on its own.
|
||||
Raise or lower the 10 MB per-asset limit with
|
||||
<tt>--single-file-max-size N</tt>; anything over it, plus audio and video, keeps
|
||||
its link. One caveat if you raise it: an inlined stylesheet becomes a
|
||||
<tt>data:</tt> URL, whose path is opaque, so an asset it referenced that stayed
|
||||
over the cap no longer resolves from inside it. Raising the cap past that asset
|
||||
embeds it too and the question goes away.<br>
|
||||
Reach for this when the file has to open for someone you cannot make assumptions
|
||||
about: it is plain HTML and needs no add-on. Reach for <tt>--mime-html</tt>
|
||||
instead when a Chromium-family browser is a given and the mirror is large: MIME
|
||||
carries text parts without the base64 tax, keeps each resource's original URL,
|
||||
and stores a shared asset once rather than re-embedding it in every page that
|
||||
uses it.</small></p>
|
||||
|
||||
<h4>HTTrack as a fetch tool</h4>
|
||||
<p><tt>httrack --get https://host/file.bin --path tmp</tt><br>
|
||||
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
|
||||
all disabled.</small></p>
|
||||
|
||||
<p><br>For the complete, always-current option list, see
|
||||
<a href="httrack.man.html">the manual page</a>. For the filter language, see
|
||||
<a href="filters.html">filters</a>; for the cache and updates, see
|
||||
<a href="cache.html">cache</a>; for the change report, see
|
||||
<a href="changes.html">changes</a>.</p>
|
||||
|
||||
<!-- doc-chrome:bottom -->
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
|
||||
|
||||
<footer>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</footer>
|
||||
<!-- /doc-chrome:bottom -->
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,120 +1,75 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<script language="javascript">
|
||||
<!--
|
||||
function FOS(host,nom,info) {
|
||||
var s;
|
||||
if (info == "") info=nom+"@"+host;
|
||||
s="mail";
|
||||
document.write("<a href='"+s+"to:"+nom+"@"+host+"'>"+info+"</a>");
|
||||
}
|
||||
// -->
|
||||
</script>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
<!-- doc-chrome:head -->
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="How to reach the HTTrack project, and who contributed to it.">
|
||||
<link rel="stylesheet" href="doc.css">
|
||||
<script src="doc.js" defer></script>
|
||||
<!-- /doc-chrome:head -->
|
||||
<title>Contact and credits</title>
|
||||
<script language="javascript">
|
||||
<!--
|
||||
function FOS(host,nom,info) {
|
||||
var s;
|
||||
if (info == "") info=nom+"@"+host;
|
||||
s="mail";
|
||||
document.write("<a href='"+s+"to:"+nom+"@"+host+"'>"+info+"</a>");
|
||||
}
|
||||
// -->
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
<!-- doc-chrome:top -->
|
||||
<a class="skip" href="#main">Skip to content</a>
|
||||
|
||||
<h2 align="center"><em>Contact Us!</em></h2>
|
||||
<header class="masthead">
|
||||
<img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier">
|
||||
<div class="tagline">Open Source offline browser</div>
|
||||
</header>
|
||||
|
||||
<div class="wrap">
|
||||
|
||||
<nav class="toc" aria-label="Documentation">
|
||||
<h2>Start here</h2>
|
||||
<ul>
|
||||
<li><a href="guide.html">Interface guide</a></li>
|
||||
<li><a href="faq.html">FAQ and troubleshooting</a></li>
|
||||
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
|
||||
</ul>
|
||||
<h2>Command line</h2>
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line guide</a></li>
|
||||
<li><a href="httrack.man.html">Option reference</a></li>
|
||||
<li><a href="filters.html">Filter syntax</a></li>
|
||||
</ul>
|
||||
<h2>Developers</h2>
|
||||
<ul>
|
||||
<li><a href="dev.html">Programming</a></li>
|
||||
<li><a href="library.html">libhttrack API</a></li>
|
||||
<li><a href="plug.html">Callbacks</a></li>
|
||||
<li><a href="scripting.html">Scripting</a></li>
|
||||
<li><a href="cache.html">Cache format</a></li>
|
||||
<li><a href="changes.html">Change report format</a></li>
|
||||
</ul>
|
||||
<h2>More</h2>
|
||||
<ul>
|
||||
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
|
||||
<li><a href="contact.html" aria-current="page" class="here">Contact and credits</a></li>
|
||||
<li><a href="index.html">All documentation</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main id="main">
|
||||
<!-- /doc-chrome:top -->
|
||||
|
||||
<h1>Contact Us!</h1>
|
||||
|
||||
<br>
|
||||
|
||||
<h3>
|
||||
<h3 id="please-visit-our-website">
|
||||
<center>
|
||||
<a href="http://www.httrack.com/" target="_blank">Please visit our website!</a>
|
||||
<br>
|
||||
@@ -127,94 +82,94 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<pre>
|
||||
Informations on this project:
|
||||
|
||||
This project has been developed by:
|
||||
Xavier Roche (xroche at httrack dot com)
|
||||
for the main engine and Windows interface
|
||||
and maintainer for v2.0 and v3.0
|
||||
Yann Philippot (yphilippot at lemel dot fr)
|
||||
for the java binary dot class parser
|
||||
David Lawrie (dalawrie at lineone dot net)
|
||||
Robert Lagadec (rlagadec at yahoo dot fr)
|
||||
for checking both English & French translations
|
||||
Juan Pablo Barrio Lera (University of León)
|
||||
for Spanish translations
|
||||
Rainer Klueting (rainer at klueting dot de)
|
||||
Bastian Gorke (bastiang at yahoo dot com)
|
||||
Rudi Ferrari (Wyando at netcologne dot de)
|
||||
Marcus Gaza (MarcusGaza at t-online dot de)
|
||||
for German translations
|
||||
Rudi Ferrari (Wyando at netcologne dot de)
|
||||
for Dutch translations
|
||||
Lukasz Jokiel (Opole University of Technology, Lukasz dot Jokiel at po dot opole dot pl)
|
||||
for Polish translations
|
||||
Rui Fernandes (CANTIC, ruiefe at mail dot malhatlantica dot pt)
|
||||
Pedro T dot Pinheiro (Universidade Nova de Lisboa-FCT, ptiago at mail dot iupi dot pt)
|
||||
for Portuguese translations
|
||||
Andrei Iliev (iliev at vitaplus dot ru)
|
||||
for Russian translations
|
||||
Witold Krakowski (wtkrak at netscape dot net )
|
||||
for Italian translations
|
||||
Jozsef Tamas Herczeg (hdodi at freemail dot hu)
|
||||
for Hungarian translation
|
||||
Paulo Neto (company at layout dot com dot br)
|
||||
for Brazilian translation
|
||||
Brook Qin (brookqwr at sina dot com)
|
||||
for simplified Chinese translation
|
||||
David Hing Cheong Hung (DAVEHUNG at mtr dot com dot hk)
|
||||
Addy Lin (addy1975 at pchome dot com dot tw)
|
||||
for traditional Chinese translation
|
||||
Jesper Bramm (bramm at get2net dot dk)
|
||||
for Danish translation
|
||||
Tõnu Virma
|
||||
for Estonian translation
|
||||
Staffan Ström (staffan at fam-strom dot org)
|
||||
for Swedish translation
|
||||
Mehmet Akif Köeoðlu (mak at ttnet dot net dot tr)
|
||||
for Turkish translation
|
||||
Aleksandar Savic (aleks at macedonia dot eu dot org)
|
||||
for Macedonian translation
|
||||
Takayoshi Nakasikiryo
|
||||
for Japanese translation
|
||||
Martin Sereday (sereday at slovanet dot sk)
|
||||
for Slovak translation
|
||||
Antonín Matìjèík (matejcik at volny dot cz)
|
||||
for Czech translation
|
||||
Andrij Shevchuk (http://programy dot com dot ua)
|
||||
for Ukrainian translation
|
||||
Tobias "Spug" Langhoff (spug_enigma at hotmail dot com)
|
||||
for Norwegian translation
|
||||
Jadran Rudeciur (jrudec at email dot si)
|
||||
for Slovenian translation
|
||||
Alin Gheorghe Miron (miron dot alin at personal dot ro)
|
||||
for Romanian translation
|
||||
Michael Papadakis (mikepap at freemail dot gr)
|
||||
for Greek translation
|
||||
|
||||
Thanks to:
|
||||
Leto Kauler (molotov at tasmail dot com)
|
||||
for the site/logos design
|
||||
|
||||
Special Thanks to:
|
||||
Patrick Ducrot & Daniel Carré (ENSI of Caen)
|
||||
for their initial support
|
||||
Fred Cohen (fc at all dot net)
|
||||
for HTTrack user's guide
|
||||
|
||||
Greetings to:
|
||||
Christian Marillat (marillat dot christian at wanadoo dot fr)
|
||||
for autoconf compliance and .deb package
|
||||
Jean-loup Gailly from gzip dot org
|
||||
for the help provided with Zlib
|
||||
Eric A dot Young (eay at cryptsoft dot com)
|
||||
for SSL cryptographic material (OpenSSL)
|
||||
|
||||
Russ Freeman from gipsymedia
|
||||
for hints on DLL dynamic load
|
||||
Paul DiLascia
|
||||
for helping to fix problems with CHtmlView
|
||||
ISMRA/Ensi of Caen
|
||||
for their initial support
|
||||
.. and all users that are using and supporting HTTrack!
|
||||
This project has been developed by:
|
||||
Xavier Roche (xroche at httrack dot com)
|
||||
for the main engine and Windows interface
|
||||
and maintainer for v2.0 and v3.0
|
||||
Yann Philippot (yphilippot at lemel dot fr)
|
||||
past contributor (java binary .class parser)
|
||||
David Lawrie (dalawrie at lineone dot net)
|
||||
Robert Lagadec (rlagadec at yahoo dot fr)
|
||||
for checking both English & French translations
|
||||
Juan Pablo Barrio Lera (University of León)
|
||||
for Spanish translations
|
||||
Rainer Klueting (rainer at klueting dot de)
|
||||
Bastian Gorke (bastiang at yahoo dot com)
|
||||
Rudi Ferrari (Wyando at netcologne dot de)
|
||||
Marcus Gaza (MarcusGaza at t-online dot de)
|
||||
for German translations
|
||||
Rudi Ferrari (Wyando at netcologne dot de)
|
||||
for Dutch translations
|
||||
Lukasz Jokiel (Opole University of Technology, Lukasz dot Jokiel at po dot opole dot pl)
|
||||
for Polish translations
|
||||
Rui Fernandes (CANTIC, ruiefe at mail dot malhatlantica dot pt)
|
||||
Pedro T dot Pinheiro (Universidade Nova de Lisboa-FCT, ptiago at mail dot iupi dot pt)
|
||||
for Portuguese translations
|
||||
Andrei Iliev (iliev at vitaplus dot ru)
|
||||
for Russian translations
|
||||
Witold Krakowski (wtkrak at netscape dot net )
|
||||
for Italian translations
|
||||
Jozsef Tamas Herczeg (hdodi at freemail dot hu)
|
||||
for Hungarian translation
|
||||
Paulo Neto (company at layout dot com dot br)
|
||||
for Brazilian translation
|
||||
Brook Qin (brookqwr at sina dot com)
|
||||
for simplified Chinese translation
|
||||
David Hing Cheong Hung (DAVEHUNG at mtr dot com dot hk)
|
||||
Addy Lin (addy1975 at pchome dot com dot tw)
|
||||
for traditional Chinese translation
|
||||
Jesper Bramm (bramm at get2net dot dk)
|
||||
for Danish translation
|
||||
Tõnu Virma
|
||||
for Estonian translation
|
||||
Staffan Ström (staffan at fam-strom dot org)
|
||||
for Swedish translation
|
||||
Mehmet Akif Köeoðlu (mak at ttnet dot net dot tr)
|
||||
for Turkish translation
|
||||
Aleksandar Savic (aleks at macedonia dot eu dot org)
|
||||
for Macedonian translation
|
||||
Takayoshi Nakasikiryo
|
||||
for Japanese translation
|
||||
Martin Sereday (sereday at slovanet dot sk)
|
||||
for Slovak translation
|
||||
Antonín Matìjèík (matejcik at volny dot cz)
|
||||
for Czech translation
|
||||
Andrij Shevchuk (http://programy dot com dot ua)
|
||||
for Ukrainian translation
|
||||
Tobias "Spug" Langhoff (spug_enigma at hotmail dot com)
|
||||
for Norwegian translation
|
||||
Jadran Rudeciur (jrudec at email dot si)
|
||||
for Slovenian translation
|
||||
Alin Gheorghe Miron (miron dot alin at personal dot ro)
|
||||
for Romanian translation
|
||||
Michael Papadakis (mikepap at freemail dot gr)
|
||||
for Greek translation
|
||||
|
||||
Thanks to:
|
||||
Leto Kauler (molotov at tasmail dot com)
|
||||
for the site/logos design
|
||||
|
||||
Special Thanks to:
|
||||
Patrick Ducrot & Daniel Carré (ENSI of Caen)
|
||||
for their initial support
|
||||
Fred Cohen (fc at all dot net)
|
||||
for HTTrack user's guide
|
||||
|
||||
Greetings to:
|
||||
Christian Marillat (marillat dot christian at wanadoo dot fr)
|
||||
for autoconf compliance and .deb package
|
||||
Jean-loup Gailly from gzip dot org
|
||||
for the help provided with Zlib
|
||||
Eric A dot Young (eay at cryptsoft dot com)
|
||||
for SSL cryptographic material (OpenSSL)
|
||||
|
||||
Russ Freeman from gipsymedia
|
||||
for hints on DLL dynamic load
|
||||
Paul DiLascia
|
||||
for helping to fix problems with CHtmlView
|
||||
ISMRA/Ensi of Caen
|
||||
for their initial support
|
||||
.. and all users that are using and supporting HTTrack!
|
||||
|
||||
</pre>
|
||||
<hr>
|
||||
@@ -225,46 +180,35 @@ ISMRA/Ensi of Caen
|
||||
You can also contact by email, but due to the large volume of messages, it is impossible
|
||||
to always respond (especially for configuration help or other configuration-related
|
||||
questions).
|
||||
<br>
|
||||
|
||||
<script language="javascript">
|
||||
<!--
|
||||
FOS('httrack.com','httrack','httrack at httrack dot com (HTTrack project)');
|
||||
document.write('<br>');
|
||||
FOS('httrack.com','roche','roche at httrack dot com (Xavier ROCHE)');
|
||||
// -->
|
||||
</script>
|
||||
<noscript>
|
||||
httrack at httrack dot com (HTTrack project)<br>
|
||||
roche at httrack dot com (Xavier ROCHE)<br>
|
||||
</noscript>
|
||||
|
||||
|
||||
<br>
|
||||
|
||||
<script language="javascript">
|
||||
<!--
|
||||
FOS('httrack.com','httrack','httrack at httrack dot com (HTTrack project)');
|
||||
document.write('<br>');
|
||||
FOS('httrack.com','roche','roche at httrack dot com (Xavier ROCHE)');
|
||||
// -->
|
||||
</script>
|
||||
<noscript>
|
||||
httrack at httrack dot com (HTTrack project)<br>
|
||||
roche at httrack dot com (Xavier ROCHE)<br>
|
||||
</noscript>
|
||||
|
||||
|
||||
<br><hr><br>
|
||||
<br>
|
||||
This program is covered by the GNU General Public License.<br>
|
||||
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2007 Xavier Roche and other contributors
|
||||
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2026 Xavier Roche and other contributors
|
||||
<br>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<!-- doc-chrome:bottom -->
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
|
||||
|
||||
<footer>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</footer>
|
||||
<!-- /doc-chrome:bottom -->
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
|
||||
|
||||
231
html/dev.html
231
html/dev.html
@@ -1,158 +1,111 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
<!-- doc-chrome:head -->
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="Using HTTrack from a script, from the callbacks, or through the libhttrack library.">
|
||||
<link rel="stylesheet" href="doc.css">
|
||||
<script src="doc.js" defer></script>
|
||||
<!-- /doc-chrome:head -->
|
||||
<title>Programming with HTTrack</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
<!-- doc-chrome:top -->
|
||||
<a class="skip" href="#main">Skip to content</a>
|
||||
|
||||
<h2 align="center"><em>HTTrack Programming page</em></h2>
|
||||
<header class="masthead">
|
||||
<img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier">
|
||||
<div class="tagline">Open Source offline browser</div>
|
||||
</header>
|
||||
|
||||
<div class="wrap">
|
||||
|
||||
<nav class="toc" aria-label="Documentation">
|
||||
<h2>On this page</h2>
|
||||
<ul>
|
||||
<li><a href="#programming">Programming</a></li>
|
||||
<li><a href="#technical-references">Technical references</a></li>
|
||||
</ul>
|
||||
<h2>Start here</h2>
|
||||
<ul>
|
||||
<li><a href="guide.html">Interface guide</a></li>
|
||||
<li><a href="faq.html">FAQ and troubleshooting</a></li>
|
||||
<li><a href="abuse.html">Using HTTrack responsibly</a></li>
|
||||
</ul>
|
||||
<h2>Command line</h2>
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line guide</a></li>
|
||||
<li><a href="httrack.man.html">Option reference</a></li>
|
||||
<li><a href="filters.html">Filter syntax</a></li>
|
||||
</ul>
|
||||
<h2>Developers</h2>
|
||||
<ul>
|
||||
<li><a href="dev.html" aria-current="page" class="here">Programming</a></li>
|
||||
<li><a href="library.html">libhttrack API</a></li>
|
||||
<li><a href="plug.html">Callbacks</a></li>
|
||||
<li><a href="scripting.html">Scripting</a></li>
|
||||
<li><a href="cache.html">Cache format</a></li>
|
||||
<li><a href="changes.html">Change report format</a></li>
|
||||
</ul>
|
||||
<h2>More</h2>
|
||||
<ul>
|
||||
<li><a href="fcguide.html">Users Guide (3.10)</a></li>
|
||||
<li><a href="contact.html">Contact and credits</a></li>
|
||||
<li><a href="index.html">All documentation</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main id="main">
|
||||
<!-- /doc-chrome:top -->
|
||||
|
||||
<h1>HTTrack Programming page</h1>
|
||||
|
||||
<br>
|
||||
|
||||
HTTrack can be used as a third-party program in batch files, or as library.
|
||||
HTTrack can be used as a third-party program in batch files, or as library.
|
||||
Depending on your needs, you may look:
|
||||
|
||||
<br><br>
|
||||
|
||||
<h3>Programming</h3>
|
||||
|
||||
<ul>
|
||||
<h3 id="programming">Programming</h3>
|
||||
|
||||
<ul>
|
||||
<li><a href="scripting.html">Simple use: scripting examples using the httrack commandline program</a></li><br>
|
||||
You can use tens of options (see <tt>httrack --help</tt>) to control precisely the mirror, and used embedded commands in httrack.
|
||||
<br><br>
|
||||
<li><a href="plug.html">More complex use: plugging external C functions to the httrack library</a></li><br>
|
||||
For advanced functions, you may have to use external C wrappers ; for example when adding advanced crawl features, such as "tuned" filestructure type
|
||||
<br><i>Important note: please read the license information of httrack before developing add-ons</i><br />
|
||||
See also the page <a href="plug_330.html">for versions prior to 3.41</a>
|
||||
<br><br>
|
||||
<li><a href="library.html">Advanced use: using the library</a></li><br>
|
||||
The library can be used to write graphical GUIs for httrack, or to run mirrors from a program.
|
||||
<br><i>Important note: please read the license information of httrack before developing add-ons</i>
|
||||
</ul>
|
||||
|
||||
<h3>Technical references</h3>
|
||||
<ul>
|
||||
<li><a href="cache.html">Cache format</a></li><br>
|
||||
HTTrack stores original HTML data and references to downloaded files in a cache, located in the hts-cache directory.
|
||||
This page describes the HTTrack cache format.
|
||||
</ul>
|
||||
|
||||
You can use tens of options (see <tt>httrack --help</tt>) to control precisely the mirror, and used embedded commands in httrack.
|
||||
<br><br>
|
||||
<li><a href="plug.html">More complex use: plugging external C functions to the httrack library</a></li><br>
|
||||
For advanced functions, you may have to use external C wrappers ; for example when adding advanced crawl features, such as "tuned" filestructure type
|
||||
<br><i>Important note: please read the license information of httrack before developing add-ons</i><br />
|
||||
See also the page <a href="plug_330.html">for versions prior to 3.41</a>
|
||||
<br><br>
|
||||
<li><a href="library.html">Advanced use: using the library</a></li><br>
|
||||
The library can be used to write graphical GUIs for httrack, or to run mirrors from a program.
|
||||
<br><i>Important note: please read the license information of httrack before developing add-ons</i>
|
||||
</ul>
|
||||
|
||||
<h3 id="technical-references">Technical references</h3>
|
||||
<ul>
|
||||
<li><a href="cache.html">Cache format</a></li><br>
|
||||
HTTrack stores original HTML data and references to downloaded files in a cache, located in the hts-cache directory.
|
||||
This page describes the HTTrack cache format.
|
||||
<li><a href="changes.html">Change report format</a></li><br>
|
||||
With --changes, HTTrack writes hts-changes.json describing what the crawl left new, changed, unchanged and gone
|
||||
compared to the previous mirror. This page describes that file.
|
||||
</ul>
|
||||
|
||||
|
||||
<br><br>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<!-- doc-chrome:bottom -->
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
<dialog id="zoom" aria-label="Enlarged image"><img src="" alt=""></dialog>
|
||||
|
||||
<footer>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</footer>
|
||||
<!-- /doc-chrome:bottom -->
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
|
||||
|
||||
341
html/doc.css
Normal file
341
html/doc.css
Normal file
@@ -0,0 +1,341 @@
|
||||
/* Shared chrome for the HTTrack documentation pages. Page-specific rules live
|
||||
beside the page (guide.css). Read from disk as often as over http, so
|
||||
everything here has to work from a file:// URL. */
|
||||
|
||||
:root {
|
||||
--field: #77b;
|
||||
--panel: #ccd;
|
||||
--accent: #99c;
|
||||
--ink: #111;
|
||||
--ink-soft: #444;
|
||||
--rule: #aab;
|
||||
--link: #00c;
|
||||
--link-hover: #c00;
|
||||
--code-bg: #e8e8f2;
|
||||
--chip: #dde;
|
||||
--shadow: rgba(0, 0, 0, .25);
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--field: #171a2b;
|
||||
--panel: #22263a;
|
||||
--accent: #3b4066;
|
||||
--ink: #e6e6ee;
|
||||
--ink-soft: #a9adc4;
|
||||
--rule: #3b4066;
|
||||
--link: #9db4ff;
|
||||
--link-hover: #ff9d9d;
|
||||
--code-bg: #14172a;
|
||||
--chip: #2e3350;
|
||||
--shadow: rgba(0, 0, 0, .6);
|
||||
}
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
/* The plain declaration before each var() is what IE11 gets: it drops the
|
||||
custom-property line it cannot parse and keeps the literal above it. */
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0 1rem 3rem;
|
||||
background: #77b;
|
||||
background: var(--field);
|
||||
color: #111;
|
||||
color: var(--ink);
|
||||
font: 1em/1.6 "Trebuchet MS", Verdana, system-ui, -apple-system, "Segoe UI", sans-serif;
|
||||
}
|
||||
|
||||
/* ---- page frame ---- */
|
||||
|
||||
.masthead, .wrap {
|
||||
max-width: 68rem;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.masthead {
|
||||
padding: .6rem 0 0;
|
||||
}
|
||||
|
||||
.masthead img { display: block; max-width: 100%; height: auto; }
|
||||
|
||||
.masthead .tagline {
|
||||
background: #000;
|
||||
color: #fff;
|
||||
font-weight: bold;
|
||||
padding: .25rem .4rem;
|
||||
margin-top: .4rem;
|
||||
}
|
||||
|
||||
.wrap {
|
||||
background: #ccd;
|
||||
background: var(--panel);
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 1.5rem;
|
||||
/* The rings the 2007 pages carried. Transparent, so it rides on --panel. */
|
||||
background-image: url(images/bg_rings.svg);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
/* Rings dropped, not inverted: their lavender is a light-panel tone. */
|
||||
.wrap { background-image: none; }
|
||||
|
||||
/* The wordmark is black on transparent, and all but vanishes on the dark field. */
|
||||
.masthead img { filter: invert(1); }
|
||||
}
|
||||
|
||||
@media (min-width: 62rem) {
|
||||
.wrap {
|
||||
display: grid;
|
||||
grid-template-columns: 14rem minmax(0, 1fr);
|
||||
gap: 2.5rem;
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
/* The hub is its own navigation, so it spans the full width. */
|
||||
.wrap.solo { display: block; }
|
||||
}
|
||||
|
||||
footer {
|
||||
max-width: 68rem;
|
||||
margin: .5rem auto 0;
|
||||
color: #eef;
|
||||
font-size: .8em;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
footer { color: var(--ink-soft); }
|
||||
}
|
||||
|
||||
/* ---- typography ---- */
|
||||
|
||||
h2, h3 { scroll-margin-top: 1rem; }
|
||||
|
||||
h1 { font-size: 1.9em; margin: 0 0 .3em; line-height: 1.2; }
|
||||
h2 { font-size: 1.45em; margin: 2.5em 0 .5em; padding-bottom: .2em; border-bottom: 2px solid var(--rule); }
|
||||
h3 { font-size: 1.15em; margin: 2em 0 .4em; }
|
||||
h2:first-child, h1 + h2 { margin-top: 1em; }
|
||||
|
||||
p { margin: 0 0 1em; }
|
||||
|
||||
a { color: #00c; color: var(--link); }
|
||||
a:hover { color: #c00; color: var(--link-hover); }
|
||||
|
||||
code, kbd, pre {
|
||||
font-family: ui-monospace, "Cascadia Mono", Consolas, "DejaVu Sans Mono", monospace;
|
||||
font-size: .9em;
|
||||
}
|
||||
|
||||
code, kbd { background: var(--code-bg); padding: .1em .35em; border-radius: 3px; }
|
||||
|
||||
/* Legacy pages carry wide code samples: scroll them, never the page. */
|
||||
pre {
|
||||
overflow-x: auto;
|
||||
max-width: 100%;
|
||||
padding: .7rem;
|
||||
background: var(--code-bg);
|
||||
border-left: 3px solid var(--accent);
|
||||
}
|
||||
|
||||
.lede { font-size: 1.08em; color: var(--ink-soft); }
|
||||
|
||||
hr { height: 1px; border: 0; background: var(--rule); margin: 2em 0; }
|
||||
|
||||
img { max-width: 100%; height: auto; }
|
||||
|
||||
/* Legacy prose carries bare URLs long enough to push a phone sideways. */
|
||||
body { overflow-wrap: break-word; }
|
||||
|
||||
/* The reference pages carry tables wider than any column: scroll the table, not
|
||||
the page. The guide's own MIME table is narrow and reads better left alone. */
|
||||
main table:not(.mime) { display: block; overflow-x: auto; }
|
||||
|
||||
/* ---- table of contents ---- */
|
||||
|
||||
.toc { font-size: .92em; }
|
||||
|
||||
@media (min-width: 62rem) {
|
||||
.toc { position: sticky; top: 1rem; max-height: calc(100vh - 2rem); overflow-y: auto; }
|
||||
}
|
||||
|
||||
.toc h2 {
|
||||
font-size: .8em;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .08em;
|
||||
margin: 0 0 .5em;
|
||||
border: 0;
|
||||
color: var(--ink-soft);
|
||||
}
|
||||
|
||||
.toc ol, .toc ul { list-style: none; margin: 0 0 1.5em; padding: 0; }
|
||||
.toc li { margin: 0; }
|
||||
|
||||
.toc a {
|
||||
display: block;
|
||||
padding: .2rem .5rem;
|
||||
text-decoration: none;
|
||||
border-left: 3px solid transparent;
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.toc a:hover { background: var(--accent); }
|
||||
.toc a[aria-current="true"] { border-left-color: var(--link); font-weight: bold; }
|
||||
.toc .here { font-weight: bold; }
|
||||
|
||||
/* ---- callouts ---- */
|
||||
|
||||
.note {
|
||||
margin: 0 0 1em;
|
||||
padding: .7rem 1rem;
|
||||
border-left: 4px solid var(--accent);
|
||||
background: var(--chip);
|
||||
font-size: .95em;
|
||||
}
|
||||
|
||||
.note > :last-child { margin-bottom: 0; }
|
||||
|
||||
/* ---- the hub ---- */
|
||||
|
||||
.hub {
|
||||
display: grid;
|
||||
gap: 1.5rem;
|
||||
margin: 2rem 0 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
@media (min-width: 46rem) {
|
||||
.hub { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
}
|
||||
|
||||
.hub > li {
|
||||
padding: 1.2rem;
|
||||
background: var(--chip);
|
||||
border-radius: .5rem;
|
||||
}
|
||||
|
||||
.hub h2 {
|
||||
margin: 0 0 .8em;
|
||||
font-size: 1.1em;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .06em;
|
||||
border: 0;
|
||||
color: var(--ink-soft);
|
||||
}
|
||||
|
||||
.hub dl { margin: 0; }
|
||||
.hub dt { margin: 0 0 .1em; font-weight: bold; }
|
||||
.hub dd { margin: 0 0 1em; font-size: .93em; color: var(--ink-soft); }
|
||||
.hub dd:last-child { margin-bottom: 0; }
|
||||
|
||||
/* Option names in the glosses are short; without this a browser breaks
|
||||
"--changes" after its hyphens. */
|
||||
.hub code { white-space: nowrap; }
|
||||
|
||||
.dated { font-weight: normal; font-size: .85em; color: var(--ink-soft); }
|
||||
|
||||
/* ---- content tables ---- */
|
||||
|
||||
/* The reference pages use bare <table> for their content tables. The guide's MIME
|
||||
table is styled beside the guide and opts out. */
|
||||
main table:not(.mime) {
|
||||
border-collapse: collapse;
|
||||
margin: 0 0 1em;
|
||||
font-size: .92em;
|
||||
}
|
||||
|
||||
main table:not(.mime) td, main table:not(.mime) th {
|
||||
border: 1px solid var(--rule);
|
||||
padding: .3em .6em;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
main table:not(.mime) th, main table:not(.mime) tr.head td {
|
||||
background: var(--accent);
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
/* ---- screenshots ---- */
|
||||
|
||||
figure {
|
||||
margin: 0 0 1.5em;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
figure img {
|
||||
border: 1px solid var(--rule);
|
||||
background: #fff;
|
||||
box-shadow: 0 2px 8px var(--shadow);
|
||||
cursor: zoom-in;
|
||||
}
|
||||
|
||||
figcaption {
|
||||
margin-top: .5em;
|
||||
font-size: .88em;
|
||||
color: var(--ink-soft);
|
||||
}
|
||||
|
||||
/* Zoom overlay. Closed <dialog> is hidden by the UA, so nothing shows without JS. */
|
||||
dialog#zoom {
|
||||
max-width: 96vw;
|
||||
max-height: 96vh;
|
||||
padding: 0;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
dialog#zoom img {
|
||||
display: block;
|
||||
max-width: 96vw;
|
||||
max-height: 96vh;
|
||||
width: auto;
|
||||
height: auto;
|
||||
cursor: zoom-out;
|
||||
}
|
||||
|
||||
dialog#zoom::backdrop { background: rgba(0, 0, 0, .8); }
|
||||
|
||||
/* ---- filter box (guide options, and the FAQ) ---- */
|
||||
|
||||
.filter {
|
||||
display: flex;
|
||||
gap: .5rem;
|
||||
align-items: center;
|
||||
margin: 0 0 1.5rem;
|
||||
}
|
||||
|
||||
.filter input {
|
||||
flex: 1 1 auto;
|
||||
padding: .5rem .7rem;
|
||||
border: 1px solid var(--rule);
|
||||
border-radius: .3rem;
|
||||
background: var(--panel);
|
||||
color: var(--ink);
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
.filter output { font-size: .88em; color: var(--ink-soft); white-space: nowrap; }
|
||||
|
||||
/* ---- misc ---- */
|
||||
|
||||
.skip {
|
||||
position: absolute;
|
||||
left: -9999px;
|
||||
}
|
||||
|
||||
.skip:focus {
|
||||
left: 1rem;
|
||||
top: 1rem;
|
||||
background: var(--panel);
|
||||
padding: .5rem;
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
@media print {
|
||||
body { background: #fff; }
|
||||
.toc, .filter { display: none; }
|
||||
.wrap { display: block; border: 0; }
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user