mirror of
https://github.com/xroche/httrack.git
synced 2026-07-09 02:16:50 +03:00
Compare commits
2 Commits
master
...
remove-dea
| Author | SHA1 | Date | |
|---|---|---|---|
|
cf483a24e4 | ||
|
|
abaea566bd |
39
.github/workflows/ci.yml
vendored
39
.github/workflows/ci.yml
vendored
@@ -278,45 +278,6 @@ jobs:
|
|||||||
if: failure()
|
if: failure()
|
||||||
run: cat tests/test-suite.log 2>/dev/null || true
|
run: cat tests/test-suite.log 2>/dev/null || true
|
||||||
|
|
||||||
# libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
|
|
||||||
# parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
|
|
||||||
# (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
|
|
||||||
# catastrophic backtracking -- and pins the regression seeds for the bugs the
|
|
||||||
# fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
|
|
||||||
fuzz:
|
|
||||||
name: fuzz (libFuzzer corpus replay, clang)
|
|
||||||
runs-on: ubuntu-24.04
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
with:
|
|
||||||
submodules: recursive
|
|
||||||
|
|
||||||
- name: Install build dependencies
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get install -y --no-install-recommends \
|
|
||||||
build-essential clang autoconf automake libtool autoconf-archive \
|
|
||||||
zlib1g-dev libssl-dev
|
|
||||||
|
|
||||||
- name: Configure (fuzzers, static)
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
autoreconf -fi
|
|
||||||
./configure CC=clang \
|
|
||||||
CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
|
|
||||||
LDFLAGS="-fsanitize=address,undefined" \
|
|
||||||
--enable-fuzzers --disable-shared
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
run: make -j"$(nproc)"
|
|
||||||
|
|
||||||
- name: Replay corpora
|
|
||||||
env:
|
|
||||||
ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
|
|
||||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
|
||||||
run: bash fuzz/run-fuzzers.sh fuzz check
|
|
||||||
|
|
||||||
# Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
|
# Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
|
||||||
# the configuration users on minimal systems build, and one libssl is not even
|
# the configuration users on minimal systems build, and one libssl is not even
|
||||||
# installed here so configure cannot silently re-enable it. The matrix above
|
# installed here so configure cannot silently re-enable it. The matrix above
|
||||||
|
|||||||
57
.github/workflows/codeql.yml
vendored
57
.github/workflows/codeql.yml
vendored
@@ -1,57 +0,0 @@
|
|||||||
# CodeQL static analysis (C). The security-extended suite covers the classes
|
|
||||||
# this codebase actually fights: overflows, tainted-size allocs, format bugs.
|
|
||||||
name: CodeQL
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [master]
|
|
||||||
pull_request:
|
|
||||||
schedule:
|
|
||||||
# Weekly re-scan of master so new/updated queries land without a push.
|
|
||||||
- cron: "17 4 * * 1"
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: codeql-${{ github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
analyze:
|
|
||||||
name: analyze (c-cpp)
|
|
||||||
runs-on: ubuntu-24.04
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
# Upload findings to the repo's code-scanning dashboard.
|
|
||||||
security-events: write
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
with:
|
|
||||||
submodules: recursive
|
|
||||||
|
|
||||||
- name: Install build dependencies
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get install -y --no-install-recommends \
|
|
||||||
build-essential autoconf automake libtool autoconf-archive \
|
|
||||||
zlib1g-dev libssl-dev
|
|
||||||
|
|
||||||
- uses: github/codeql-action/init@v4
|
|
||||||
with:
|
|
||||||
languages: c-cpp
|
|
||||||
build-mode: manual
|
|
||||||
queries: security-extended
|
|
||||||
|
|
||||||
# Manual build: CodeQL traces the compiler, so build exactly what ships.
|
|
||||||
- name: Build
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
autoreconf -fi
|
|
||||||
./configure
|
|
||||||
make -j"$(nproc)"
|
|
||||||
|
|
||||||
- uses: github/codeql-action/analyze@v4
|
|
||||||
with:
|
|
||||||
category: "/language:c-cpp"
|
|
||||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -34,9 +34,8 @@ Makefile
|
|||||||
*.so.*
|
*.so.*
|
||||||
*.a
|
*.a
|
||||||
|
|
||||||
# make dist output; dist/ holds httrack-gh-release staging artifacts.
|
# make dist output.
|
||||||
/httrack-*.tar.gz
|
/httrack-*.tar.gz
|
||||||
/dist/
|
|
||||||
|
|
||||||
# Editor / autotools backup files.
|
# Editor / autotools backup files.
|
||||||
*~
|
*~
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
|
|
||||||
SUBDIRS = src man m4 libtest templates lang html tests fuzz
|
SUBDIRS = src man m4 libtest templates lang html tests
|
||||||
|
|
||||||
ACLOCAL_AMFLAGS = -I m4
|
ACLOCAL_AMFLAGS = -I m4
|
||||||
|
|
||||||
|
|||||||
42
configure.ac
42
configure.ac
@@ -102,8 +102,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
|
|||||||
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
|
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
|
||||||
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
|
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
|
||||||
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
|
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
|
||||||
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
|
AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
|
||||||
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
|
|
||||||
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
|
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
|
||||||
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
|
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
|
||||||
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
|
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
|
||||||
@@ -111,13 +110,6 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
|
|||||||
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
|
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
|
||||||
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
|
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
|
||||||
|
|
||||||
# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
|
|
||||||
# sanitizer builds, whose interceptors want the unfortified calls.
|
|
||||||
case "$CFLAGS" in
|
|
||||||
*-fsanitize=*) ;;
|
|
||||||
*) AX_ADD_FORTIFY_SOURCE ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Force libc back into DT_NEEDED for libraries that reach it only through
|
# Force libc back into DT_NEEDED for libraries that reach it only through
|
||||||
# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
|
# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
|
||||||
# linker; Apple ld rejects these flags and links libSystem unconditionally.
|
# linker; Apple ld rejects these flags and links libSystem unconditionally.
|
||||||
@@ -318,37 +310,6 @@ AC_ARG_ENABLE([online-unit-tests],
|
|||||||
])
|
])
|
||||||
AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
|
AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
|
||||||
|
|
||||||
## libFuzzer harnesses (fuzz/); requires clang
|
|
||||||
AC_MSG_CHECKING(whether to build fuzzers)
|
|
||||||
AC_ARG_ENABLE([fuzzers],
|
|
||||||
[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
|
|
||||||
[
|
|
||||||
case "${enableval}" in
|
|
||||||
no|yes)
|
|
||||||
fuzzers=$enableval
|
|
||||||
AC_MSG_RESULT($enableval)
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
],
|
|
||||||
[
|
|
||||||
fuzzers=no
|
|
||||||
AC_MSG_RESULT(no)
|
|
||||||
])
|
|
||||||
if test x"$fuzzers" = x"yes"; then
|
|
||||||
# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
|
|
||||||
AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
|
|
||||||
[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
|
|
||||||
[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
|
|
||||||
# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
|
|
||||||
if test x"$enable_shared" != x"no"; then
|
|
||||||
AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
|
|
||||||
|
|
||||||
# Final output
|
# Final output
|
||||||
AC_CONFIG_FILES([
|
AC_CONFIG_FILES([
|
||||||
Makefile
|
Makefile
|
||||||
@@ -360,6 +321,5 @@ lang/Makefile
|
|||||||
html/Makefile
|
html/Makefile
|
||||||
libtest/Makefile
|
libtest/Makefile
|
||||||
tests/Makefile
|
tests/Makefile
|
||||||
fuzz/Makefile
|
|
||||||
])
|
])
|
||||||
AC_OUTPUT
|
AC_OUTPUT
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
|
|
||||||
if FUZZERS
|
|
||||||
noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
|
|
||||||
fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
|
|
||||||
endif
|
|
||||||
|
|
||||||
AM_CPPFLAGS = \
|
|
||||||
@DEFAULT_CFLAGS@ \
|
|
||||||
@THREADS_CFLAGS@ \
|
|
||||||
@V6_FLAG@ \
|
|
||||||
@LFS_FLAG@ \
|
|
||||||
-I$(top_srcdir)/src \
|
|
||||||
-I$(top_srcdir)/src/coucal
|
|
||||||
|
|
||||||
# Static-link libhttrack.la: the internal symbols are hidden in the .so.
|
|
||||||
AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
|
|
||||||
LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
|
|
||||||
|
|
||||||
fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
|
|
||||||
fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
|
|
||||||
fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
|
|
||||||
fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
|
|
||||||
fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
|
|
||||||
fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
|
|
||||||
fuzz_url_SOURCES = fuzz-url.c fuzz.h
|
|
||||||
fuzz_header_SOURCES = fuzz-header.c fuzz.h
|
|
||||||
fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
|
|
||||||
|
|
||||||
# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
|
|
||||||
EXTRA_DIST = README.md run-fuzzers.sh \
|
|
||||||
corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
|
|
||||||
corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
|
|
||||||
corpus/idna/idna.txt corpus/idna/unicode.txt \
|
|
||||||
corpus/idna/regress-multilabel-leak.txt \
|
|
||||||
corpus/entities/entities.txt \
|
|
||||||
corpus/unescape/percent.txt \
|
|
||||||
corpus/filters/filter.bin corpus/filters/filter-size.bin \
|
|
||||||
corpus/filters/regress-empty-subject-unique.bin \
|
|
||||||
corpus/url/http-url.txt corpus/url/relative-path.txt \
|
|
||||||
corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
|
|
||||||
corpus/header/full-response.txt corpus/header/redirect.txt \
|
|
||||||
corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
|
|
||||||
corpus/cachendx/regress-overadvance.bin \
|
|
||||||
corpus/cachendx/regress-truncated-entry.bin
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
# Fuzzing httrack
|
|
||||||
|
|
||||||
libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
./bootstrap
|
|
||||||
mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
|
|
||||||
CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
|
|
||||||
LDFLAGS="-fsanitize=address,undefined" \
|
|
||||||
bash /path/to/httrack/configure --enable-fuzzers --disable-shared
|
|
||||||
make
|
|
||||||
bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60 # 60s per target
|
|
||||||
```
|
|
||||||
|
|
||||||
Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
8
|
|
||||||
CACHE-1.1
|
|
||||||
28
|
|
||||||
Mon, 01 Jan 2024 00:00:00 GMT
|
|
||||||
www.example.com
|
|
||||||
/index.html
|
|
||||||
123
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
3
|
|
||||||
1.0
|
|
||||||
www.example.com
|
|
||||||
/page
|
|
||||||
5
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
32768
|
|
||||||
CACHE-1.1
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
8
|
|
||||||
CACHE-1.1
|
|
||||||
1
|
|
||||||
x
|
|
||||||
www.example.com
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
café naďve ¤
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
コンピュータ
|
|
||||||
Binary file not shown.
@@ -1 +0,0 @@
|
|||||||
&<>A☃é¬arealentity;�
|
|
||||||
Binary file not shown.
Binary file not shown.
@@ -1 +0,0 @@
|
|||||||
**((
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
HTTP/1.1 200 OK
|
|
||||||
Content-Type: text/html; charset=utf-8
|
|
||||||
Content-Length: 1234
|
|
||||||
Content-Encoding: gzip
|
|
||||||
Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
|
|
||||||
Etag: "abc"
|
|
||||||
Location: http://example.com/x
|
|
||||||
Set-Cookie: ID=42; path=/; domain=.example.com
|
|
||||||
Content-Range: bytes 0-99/100
|
|
||||||
Transfer-Encoding: chunked
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
HTTP/1.0 301 Moved
|
|
||||||
Location: /elsewhere
|
|
||||||
Content-Disposition: attachment; filename="a.pdf"
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
xn--bcher-kva.example.com
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
büchev.例åbücheple
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
bücher.例子.example
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
<html><head><meta charset="utf-8"></head><body>x</body></html>
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
%41%zz%%20%c3%a9+%2e%2e%2f
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
file://
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
ftp://ftp.example.com/pub/../file.txt
|
|
||||||
@@ -1,66 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 2026 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
|
|
||||||
walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
|
|
||||||
Mirrors the loader in cache_readex_new (htscache.c). */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htscache.h"
|
|
||||||
#include "htslib.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
char *buf = fuzz_strdup(data, size);
|
|
||||||
const char *const end = buf + size;
|
|
||||||
char firstline[256];
|
|
||||||
char *a = buf;
|
|
||||||
|
|
||||||
/* header: two length-prefixed fields (version, last-modified) */
|
|
||||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
|
||||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
|
||||||
|
|
||||||
/* body: newline-delimited host/file/position triples. The coucal insert
|
|
||||||
the real loader ends with is a separate library's concern; this harness
|
|
||||||
targets the length-prefixed scan that must stay inside the buffer. */
|
|
||||||
while (a != NULL && a < end) {
|
|
||||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
|
||||||
char linepos[256];
|
|
||||||
int pos;
|
|
||||||
|
|
||||||
a = strchr(a + 1, '\n');
|
|
||||||
if (a == NULL)
|
|
||||||
break;
|
|
||||||
a++;
|
|
||||||
a += cache_binput(a, end, line, HTS_URLMAXSIZE);
|
|
||||||
a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
|
|
||||||
a += cache_binput(a, end, linepos, 200);
|
|
||||||
sscanf(linepos, "%d", &pos);
|
|
||||||
(void) pos;
|
|
||||||
}
|
|
||||||
|
|
||||||
freet(buf);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
|
|
||||||
UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htscharset.h"
|
|
||||||
|
|
||||||
static const char *const charsets[] = {
|
|
||||||
"utf-8", "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
|
|
||||||
"us-ascii", "shift_jis", "euc-jp", "iso-2022-jp", "gb2312",
|
|
||||||
"big5", "euc-kr", "koi8-r", "utf-16", "unknown-charset",
|
|
||||||
};
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
const char *charset;
|
|
||||||
char *s;
|
|
||||||
|
|
||||||
if (size == 0)
|
|
||||||
return 0;
|
|
||||||
charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
|
|
||||||
data++, size--;
|
|
||||||
s = fuzz_strdup(data, size);
|
|
||||||
|
|
||||||
{
|
|
||||||
char *utf8 = hts_convertStringToUTF8(s, size, charset);
|
|
||||||
freet(utf8);
|
|
||||||
}
|
|
||||||
{
|
|
||||||
char *enc = hts_convertStringFromUTF8(s, size, charset);
|
|
||||||
freet(enc);
|
|
||||||
}
|
|
||||||
{
|
|
||||||
size_t nChars = 0;
|
|
||||||
hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
|
|
||||||
|
|
||||||
if (ucs != NULL) {
|
|
||||||
char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
|
|
||||||
freet(back);
|
|
||||||
freet(ucs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
{
|
|
||||||
size_t i = 0;
|
|
||||||
|
|
||||||
while (i < size) {
|
|
||||||
hts_UCS4 uc = 0;
|
|
||||||
const size_t nr = hts_readUTF8(s + i, size - i, &uc);
|
|
||||||
char out[8];
|
|
||||||
|
|
||||||
if (nr == 0)
|
|
||||||
break;
|
|
||||||
hts_writeUTF8(uc, out, sizeof(out));
|
|
||||||
i += nr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
freet(s);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
|
|
||||||
destination size, so truncation bounds get exercised too. */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htsencoding.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
static const size_t dsizes[] = {1, 2, 8, 64, 4096};
|
|
||||||
size_t dsize;
|
|
||||||
char *src, *dest;
|
|
||||||
|
|
||||||
if (size == 0)
|
|
||||||
return 0;
|
|
||||||
dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
|
|
||||||
data++, size--;
|
|
||||||
src = fuzz_strdup(data, size);
|
|
||||||
dest = malloct(dsize);
|
|
||||||
|
|
||||||
(void) hts_unescapeEntities(src, dest, dsize);
|
|
||||||
(void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
|
|
||||||
|
|
||||||
freet(dest);
|
|
||||||
freet(src);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
|
|
||||||
here). Input splits on the first NUL: pattern, then subject string. */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htsfilters.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
char *buf = fuzz_strdup(data, size);
|
|
||||||
const char *joker = buf;
|
|
||||||
const uint8_t *sep = memchr(data, '\0', size);
|
|
||||||
/* subject in its own allocation so ASan bounds it apart from the pattern */
|
|
||||||
char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
|
|
||||||
: fuzz_strdup(data + size, 0);
|
|
||||||
|
|
||||||
(void) strjoker(nom, joker, NULL, NULL);
|
|
||||||
{
|
|
||||||
LLint sz = (LLint) size;
|
|
||||||
int size_flag = 0;
|
|
||||||
|
|
||||||
(void) strjoker(nom, joker, &sz, &size_flag);
|
|
||||||
}
|
|
||||||
(void) strjokerfind(nom, joker);
|
|
||||||
{
|
|
||||||
char *filter = malloct(strlen(joker) + 2);
|
|
||||||
char *filters[1];
|
|
||||||
LLint sz = (LLint) size;
|
|
||||||
int size_flag = 0, depth = 0;
|
|
||||||
|
|
||||||
filter[0] = '-';
|
|
||||||
memcpy(filter + 1, joker, strlen(joker) + 1);
|
|
||||||
filters[0] = filter;
|
|
||||||
(void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
|
|
||||||
freet(filter);
|
|
||||||
}
|
|
||||||
|
|
||||||
freet(nom);
|
|
||||||
freet(buf);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 2026 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
|
|
||||||
status line, treathead on each following header. Both consume raw bytes
|
|
||||||
off the wire and copy fields into fixed htsblk buffers; treathead also
|
|
||||||
mutates its line in place and drives cookie parsing. */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htslib.h"
|
|
||||||
#include "htsbauth.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
char *buf = fuzz_strdup(data, size);
|
|
||||||
htsblk r;
|
|
||||||
t_cookie *cookie = calloct(1, sizeof(*cookie));
|
|
||||||
char *line = malloct(size + 1);
|
|
||||||
char *p = buf;
|
|
||||||
int first = 1;
|
|
||||||
|
|
||||||
memset(&r, 0, sizeof(r));
|
|
||||||
cookie->max_len = (int) sizeof(cookie->data);
|
|
||||||
r.location = malloct(HTS_URLMAXSIZE * 2);
|
|
||||||
r.location[0] = '\0';
|
|
||||||
|
|
||||||
/* feed one header line at a time, as the receive loop does */
|
|
||||||
while (p != NULL && *p != '\0') {
|
|
||||||
char *nl = strchr(p, '\n');
|
|
||||||
size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
|
|
||||||
|
|
||||||
memcpy(line, p, n);
|
|
||||||
line[n] = '\0';
|
|
||||||
if (first) {
|
|
||||||
treatfirstline(&r, line);
|
|
||||||
first = 0;
|
|
||||||
} else {
|
|
||||||
treathead(cookie, "www.example.com", "/", &r, line);
|
|
||||||
}
|
|
||||||
p = (nl != NULL) ? nl + 1 : NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
freet(r.location);
|
|
||||||
freet(line);
|
|
||||||
freet(cookie);
|
|
||||||
freet(buf);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htscharset.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
char *s = fuzz_strdup(data, size);
|
|
||||||
|
|
||||||
{
|
|
||||||
char *idna = hts_convertStringUTF8ToIDNA(s, size);
|
|
||||||
freet(idna);
|
|
||||||
}
|
|
||||||
{
|
|
||||||
char *utf8 = hts_convertStringIDNAToUTF8(s, size);
|
|
||||||
freet(utf8);
|
|
||||||
}
|
|
||||||
(void) hts_isStringIDNA(s, size);
|
|
||||||
|
|
||||||
freet(s);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
|
|
||||||
<meta> charset declaration. */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htscharset.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
char *html = fuzz_strdup(data, size);
|
|
||||||
char *charset = hts_getCharsetFromMeta(html, size);
|
|
||||||
|
|
||||||
freet(charset);
|
|
||||||
freet(html);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
|
|
||||||
output buffer size, so the bounded-copy contract is exercised. */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "httrack-library.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
static const size_t bsizes[] = {1, 2, 16, 256, 8192};
|
|
||||||
size_t bsize;
|
|
||||||
char *s, *catbuff;
|
|
||||||
|
|
||||||
if (size == 0)
|
|
||||||
return 0;
|
|
||||||
bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
|
|
||||||
data++, size--;
|
|
||||||
s = fuzz_strdup(data, size);
|
|
||||||
catbuff = malloct(bsize);
|
|
||||||
|
|
||||||
(void) unescape_http(catbuff, bsize, s);
|
|
||||||
(void) unescape_http_unharm(catbuff, bsize, s, 0);
|
|
||||||
(void) unescape_http_unharm(catbuff, bsize, s, 1);
|
|
||||||
unescape_amp(s);
|
|
||||||
|
|
||||||
freet(catbuff);
|
|
||||||
freet(s);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
|
|
||||||
is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
|
|
||||||
in place. */
|
|
||||||
#include "fuzz.h"
|
|
||||||
#include "htscore.h"
|
|
||||||
#include "htslib.h"
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
||||||
char *s = fuzz_strdup(data, size);
|
|
||||||
lien_adrfil *af = calloct(1, sizeof(*af));
|
|
||||||
/* fil_simplifie rewrites in place and may grow an empty path to "./" */
|
|
||||||
char *path = malloct(size + 3);
|
|
||||||
|
|
||||||
(void) ident_url_absolute(s, af);
|
|
||||||
memcpy(path, s, size + 1);
|
|
||||||
fil_simplifie(path);
|
|
||||||
|
|
||||||
freet(path);
|
|
||||||
freet(af);
|
|
||||||
freet(s);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
51
fuzz/fuzz.h
51
fuzz/fuzz.h
@@ -1,51 +0,0 @@
|
|||||||
/* ------------------------------------------------------------ */
|
|
||||||
/*
|
|
||||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
|
||||||
Copyright (C) 1998 Xavier Roche and other contributors
|
|
||||||
|
|
||||||
SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
|
||||||
addresses or to collect any other private information about people. Doing so
|
|
||||||
would dishonor our work and waste the many hours we have spent on it.
|
|
||||||
|
|
||||||
Please visit our Website: http://www.httrack.com
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Shared helpers for the libFuzzer harnesses. */
|
|
||||||
#ifndef FUZZ_H
|
|
||||||
#define FUZZ_H
|
|
||||||
|
|
||||||
#define HTS_INTERNAL_BYTECODE
|
|
||||||
|
|
||||||
#include <stddef.h>
|
|
||||||
#include <stdint.h>
|
|
||||||
#include <string.h>
|
|
||||||
|
|
||||||
#include "htsbase.h"
|
|
||||||
|
|
||||||
/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
|
|
||||||
static char *fuzz_strdup(const uint8_t *data, size_t size) {
|
|
||||||
char *s = malloct(size + 1);
|
|
||||||
|
|
||||||
memcpy(s, data, size);
|
|
||||||
s[size] = '\0';
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Drive every built harness against its seed corpus.
|
|
||||||
# run-fuzzers.sh <build-fuzz-dir> check deterministic replay (CI smoke)
|
|
||||||
# run-fuzzers.sh <build-fuzz-dir> [seconds] timed mutation run (discovery)
|
|
||||||
# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
|
|
||||||
# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
srcdir=$(cd "$(dirname "$0")" && pwd)
|
|
||||||
bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
|
|
||||||
mode=${2:-20}
|
|
||||||
|
|
||||||
status=0
|
|
||||||
for f in "$bld"/fuzz-*; do
|
|
||||||
if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
|
|
||||||
case "$f" in *.o | *.c | *.dSYM) continue ;; esac
|
|
||||||
name=$(basename "$f")
|
|
||||||
corpus="$srcdir/corpus/${name#fuzz-}"
|
|
||||||
if [ "$mode" = "check" ]; then
|
|
||||||
echo "=== $name (replay) ==="
|
|
||||||
[ -d "$corpus" ] || continue
|
|
||||||
if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
|
|
||||||
echo "*** $name FAILED on its corpus" >&2
|
|
||||||
status=1
|
|
||||||
fi
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
work=$(mktemp -d)
|
|
||||||
args=("$work")
|
|
||||||
[ -d "$corpus" ] && args+=("$corpus")
|
|
||||||
echo "=== $name (${mode}s) ==="
|
|
||||||
if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
|
|
||||||
-artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
|
|
||||||
echo "*** $name FAILED; artifacts:" >&2
|
|
||||||
ls -l "$work" >&2
|
|
||||||
status=1
|
|
||||||
else
|
|
||||||
rm -rf "$work"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
exit $status
|
|
||||||
@@ -1,119 +0,0 @@
|
|||||||
# ===========================================================================
|
|
||||||
# https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
|
|
||||||
# ===========================================================================
|
|
||||||
#
|
|
||||||
# SYNOPSIS
|
|
||||||
#
|
|
||||||
# AX_ADD_FORTIFY_SOURCE
|
|
||||||
#
|
|
||||||
# DESCRIPTION
|
|
||||||
#
|
|
||||||
# Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
|
|
||||||
# redefinition warnings, other cpp warnings or linker. Some distributions
|
|
||||||
# (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
|
|
||||||
# their compilers, leading to unnecessary warnings in the form of
|
|
||||||
#
|
|
||||||
# <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
|
|
||||||
# <built-in>: note: this is the location of the previous definition
|
|
||||||
#
|
|
||||||
# which is a problem if -Werror is enabled. This macro checks whether
|
|
||||||
# _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
|
|
||||||
# to CPPFLAGS.
|
|
||||||
#
|
|
||||||
# Newer mingw-w64 msys2 package comes with a bug in
|
|
||||||
# headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
|
|
||||||
# and would need -lssp or -fstack-protector. See
|
|
||||||
# https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
|
|
||||||
# link it.
|
|
||||||
#
|
|
||||||
# LICENSE
|
|
||||||
#
|
|
||||||
# Copyright (c) 2017 David Seifert <soap@gentoo.org>
|
|
||||||
# Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
|
|
||||||
#
|
|
||||||
# Copying and distribution of this file, with or without modification, are
|
|
||||||
# permitted in any medium without royalty provided the copyright notice
|
|
||||||
# and this notice are preserved. This file is offered as-is, without any
|
|
||||||
# warranty.
|
|
||||||
|
|
||||||
#serial 10
|
|
||||||
|
|
||||||
AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
|
|
||||||
ac_save_cflags=$CFLAGS
|
|
||||||
ac_cwerror_flag=yes
|
|
||||||
AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
|
|
||||||
ax_add_fortify_3_failed=
|
|
||||||
AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
|
|
||||||
AC_LINK_IFELSE([
|
|
||||||
AC_LANG_PROGRAM([],
|
|
||||||
[[
|
|
||||||
#ifndef _FORTIFY_SOURCE
|
|
||||||
return 0;
|
|
||||||
#else
|
|
||||||
_FORTIFY_SOURCE_already_defined;
|
|
||||||
#endif
|
|
||||||
]]
|
|
||||||
)],
|
|
||||||
AC_LINK_IFELSE([
|
|
||||||
AC_LANG_SOURCE([[
|
|
||||||
#define _FORTIFY_SOURCE 3
|
|
||||||
#include <string.h>
|
|
||||||
int main(void) {
|
|
||||||
char *s = " ";
|
|
||||||
strcpy(s, "x");
|
|
||||||
return strlen(s)-1;
|
|
||||||
}
|
|
||||||
]]
|
|
||||||
)],
|
|
||||||
[
|
|
||||||
AC_MSG_RESULT([yes])
|
|
||||||
CFLAGS=$ac_save_cflags
|
|
||||||
CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
|
|
||||||
], [
|
|
||||||
AC_MSG_RESULT([no])
|
|
||||||
ax_add_fortify_3_failed=1
|
|
||||||
],
|
|
||||||
),
|
|
||||||
[
|
|
||||||
AC_MSG_RESULT([no])
|
|
||||||
ax_add_fortify_3_failed=1
|
|
||||||
])
|
|
||||||
if test -n "$ax_add_fortify_3_failed"
|
|
||||||
then
|
|
||||||
AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
|
|
||||||
AC_LINK_IFELSE([
|
|
||||||
AC_LANG_PROGRAM([],
|
|
||||||
[[
|
|
||||||
#ifndef _FORTIFY_SOURCE
|
|
||||||
return 0;
|
|
||||||
#else
|
|
||||||
_FORTIFY_SOURCE_already_defined;
|
|
||||||
#endif
|
|
||||||
]]
|
|
||||||
)],
|
|
||||||
AC_LINK_IFELSE([
|
|
||||||
AC_LANG_SOURCE([[
|
|
||||||
#define _FORTIFY_SOURCE 2
|
|
||||||
#include <string.h>
|
|
||||||
int main(void) {
|
|
||||||
char *s = " ";
|
|
||||||
strcpy(s, "x");
|
|
||||||
return strlen(s)-1;
|
|
||||||
}
|
|
||||||
]]
|
|
||||||
)],
|
|
||||||
[
|
|
||||||
AC_MSG_RESULT([yes])
|
|
||||||
CFLAGS=$ac_save_cflags
|
|
||||||
CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
|
|
||||||
], [
|
|
||||||
AC_MSG_RESULT([no])
|
|
||||||
CFLAGS=$ac_save_cflags
|
|
||||||
],
|
|
||||||
),
|
|
||||||
[
|
|
||||||
AC_MSG_RESULT([no])
|
|
||||||
CFLAGS=$ac_save_cflags
|
|
||||||
])
|
|
||||||
fi
|
|
||||||
])
|
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
.\"
|
.\"
|
||||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
.TH httrack 1 "07 July 2026" "httrack website copier"
|
.TH httrack 1 "27 June 2026" "httrack website copier"
|
||||||
.SH NAME
|
.SH NAME
|
||||||
httrack \- offline browser : copy websites to a local directory
|
httrack \- offline browser : copy websites to a local directory
|
||||||
.SH SYNOPSIS
|
.SH SYNOPSIS
|
||||||
@@ -51,7 +51,6 @@ httrack \- offline browser : copy websites to a local directory
|
|||||||
[ \fB\-%T, \-\-utf8\-conversion\fR ]
|
[ \fB\-%T, \-\-utf8\-conversion\fR ]
|
||||||
[ \fB\-bN, \-\-cookies[=N]\fR ]
|
[ \fB\-bN, \-\-cookies[=N]\fR ]
|
||||||
[ \fB\-%K, \-\-cookies\-file\fR ]
|
[ \fB\-%K, \-\-cookies\-file\fR ]
|
||||||
[ \fB\-%Y, \-\-why\fR ]
|
|
||||||
[ \fB\-u, \-\-check\-type[=N]\fR ]
|
[ \fB\-u, \-\-check\-type[=N]\fR ]
|
||||||
[ \fB\-j, \-\-parse\-java[=N]\fR ]
|
[ \fB\-j, \-\-parse\-java[=N]\fR ]
|
||||||
[ \fB\-sN, \-\-robots[=N]\fR ]
|
[ \fB\-sN, \-\-robots[=N]\fR ]
|
||||||
@@ -219,8 +218,6 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
|
|||||||
accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
|
accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
|
||||||
.IP \-%K
|
.IP \-%K
|
||||||
load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
|
load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
|
||||||
.IP \-%Y
|
|
||||||
explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
|
|
||||||
.IP \-u
|
.IP \-u
|
||||||
check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
|
check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
|
||||||
.IP \-j
|
.IP \-j
|
||||||
|
|||||||
@@ -114,8 +114,6 @@ const char *hts_optalias[][4] = {
|
|||||||
"strip [host/pattern=]key1,key2,... from URLs"},
|
"strip [host/pattern=]key1,key2,... from URLs"},
|
||||||
{"cookies-file", "-%K", "param1",
|
{"cookies-file", "-%K", "param1",
|
||||||
"load extra cookies from a Netscape cookies.txt"},
|
"load extra cookies from a Netscape cookies.txt"},
|
||||||
{"why", "-%Y", "param1",
|
|
||||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
|
||||||
{"pause", "-%G", "param1",
|
{"pause", "-%G", "param1",
|
||||||
"random pause of MIN[:MAX] seconds between files"},
|
"random pause of MIN[:MAX] seconds between files"},
|
||||||
{"generate-errors", "-o", "single", ""},
|
{"generate-errors", "-o", "single", ""},
|
||||||
|
|||||||
@@ -1938,17 +1938,15 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
|||||||
char linepos[256];
|
char linepos[256];
|
||||||
int pos;
|
int pos;
|
||||||
|
|
||||||
const char *const end = cache->use + buffl;
|
while((a != NULL) && (a < (cache->use + buffl))) {
|
||||||
|
|
||||||
while ((a != NULL) && (a < end)) {
|
|
||||||
a = strchr(a + 1, '\n'); /* start of line */
|
a = strchr(a + 1, '\n'); /* start of line */
|
||||||
if (a) {
|
if (a) {
|
||||||
a++;
|
a++;
|
||||||
/* read "host/file" */
|
/* read "host/file" */
|
||||||
a += cache_binput(a, end, line, HTS_URLMAXSIZE);
|
a += binput(a, line, HTS_URLMAXSIZE);
|
||||||
a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
|
a += binput(a, line + strlen(line), HTS_URLMAXSIZE);
|
||||||
/* read position */
|
/* read position */
|
||||||
a += cache_binput(a, end, linepos, 200);
|
a += binput(a, linepos, 200);
|
||||||
sscanf(linepos, "%d", &pos);
|
sscanf(linepos, "%d", &pos);
|
||||||
coucal_add(cache->hashtable, line, pos);
|
coucal_add(cache->hashtable, line, pos);
|
||||||
}
|
}
|
||||||
@@ -2293,40 +2291,23 @@ int cache_brstr(char *adr, char *s, size_t s_size) {
|
|||||||
char buff[256 + 4];
|
char buff[256 + 4];
|
||||||
|
|
||||||
off = binput(adr, buff, 256);
|
off = binput(adr, buff, 256);
|
||||||
/* binput stops at the buffer's terminating NUL; a value can only follow a
|
adr += off;
|
||||||
real line terminator, so never step past end-of-buffer. */
|
sscanf(buff, "%d", &i);
|
||||||
if (adr[off - 1] == '\0') {
|
if (i < 0 || i > 32768) /* guard a corrupt length */
|
||||||
s[0] = '\0';
|
|
||||||
return off - 1;
|
|
||||||
}
|
|
||||||
/* an empty/non-numeric field leaves i unset: treat as length 0 */
|
|
||||||
if (sscanf(buff, "%d", &i) != 1 || i < 0 || i > 32768)
|
|
||||||
i = 0;
|
i = 0;
|
||||||
if (i > 0) {
|
if (i > 0) {
|
||||||
/* A corrupt/truncated cache may declare a length past the buffer end;
|
/* copy at most s_size-1 bytes; advance past the full field regardless */
|
||||||
bound both the copy and the advance to the bytes actually present. */
|
const size_t store = (size_t) i < s_size ? (size_t) i : s_size - 1;
|
||||||
const size_t avail = strnlen(adr + off, (size_t) i);
|
|
||||||
const size_t store = avail < s_size ? avail : s_size - 1;
|
|
||||||
|
|
||||||
memcpy(s, adr + off, store);
|
strncpy(s, adr, store);
|
||||||
s[store] = '\0';
|
s[store] = '\0';
|
||||||
off += (int) avail;
|
|
||||||
} else {
|
} else {
|
||||||
s[0] = '\0';
|
s[0] = '\0';
|
||||||
}
|
}
|
||||||
|
off += i;
|
||||||
return off;
|
return off;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* binput bounded to a NUL-terminated buffer: refuse to start a read at or
|
|
||||||
past `end`, so a prior over-advance can't walk a cache-index parse OOB. */
|
|
||||||
int cache_binput(char *adr, const char *end, char *s, int max) {
|
|
||||||
if (adr >= end) {
|
|
||||||
s[0] = '\0';
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return binput(adr, s, max);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* idem, mais en int */
|
/* idem, mais en int */
|
||||||
int cache_brint(char *adr, int *i) {
|
int cache_brint(char *adr, int *i) {
|
||||||
char s[256];
|
char s[256];
|
||||||
|
|||||||
@@ -97,8 +97,6 @@ int cache_readdata(cache_back * cache, const char *str1, const char *str2,
|
|||||||
void cache_rstr(FILE *fp, char *s, size_t s_size);
|
void cache_rstr(FILE *fp, char *s, size_t s_size);
|
||||||
char *cache_rstr_addr(FILE * fp);
|
char *cache_rstr_addr(FILE * fp);
|
||||||
int cache_brstr(char *adr, char *s, size_t s_size);
|
int cache_brstr(char *adr, char *s, size_t s_size);
|
||||||
/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
|
|
||||||
int cache_binput(char *adr, const char *end, char *s, int max);
|
|
||||||
int cache_brint(char *adr, int *i);
|
int cache_brint(char *adr, int *i);
|
||||||
void cache_rint(FILE * fp, int *i);
|
void cache_rint(FILE * fp, int *i);
|
||||||
void cache_rLLint(FILE * fp, LLint * i);
|
void cache_rLLint(FILE * fp, LLint * i);
|
||||||
|
|||||||
@@ -910,19 +910,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
|
|||||||
/* Reader: can read bytes up to j */
|
/* Reader: can read bytes up to j */
|
||||||
#define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
|
#define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
|
||||||
|
|
||||||
/* Writer: a malformed sequence abandons the encode (NULL) */
|
/* Writer: upon error, return FFFD (replacement character) */
|
||||||
#define WR(C) \
|
#define WR(C) do { \
|
||||||
do { \
|
if ((C) != -1) { \
|
||||||
if ((C) != -1) { \
|
/* copy character */ \
|
||||||
/* copy character */ \
|
assertf(segOutputSize < segSize); \
|
||||||
assertf(segOutputSize < segSize); \
|
segInt[segOutputSize++] = (C); \
|
||||||
segInt[segOutputSize++] = (C); \
|
} \
|
||||||
} else { \
|
/* In case of error, abort. */ \
|
||||||
freet(segInt); \
|
else { \
|
||||||
FREE_BUFFER(); \
|
FREE_BUFFER(); \
|
||||||
return NULL; \
|
return NULL; \
|
||||||
} \
|
} \
|
||||||
} while (0)
|
} while(0)
|
||||||
|
|
||||||
/* Read/Write Unicode character. */
|
/* Read/Write Unicode character. */
|
||||||
READ_UNICODE(RD, WR);
|
READ_UNICODE(RD, WR);
|
||||||
@@ -1144,7 +1144,7 @@ int hts_isStringUTF8(const char *s, size_t size) {
|
|||||||
/* Reader: can read bytes up to j */
|
/* Reader: can read bytes up to j */
|
||||||
#define RD ( i < size ? data[i++] : -1 )
|
#define RD ( i < size ? data[i++] : -1 )
|
||||||
|
|
||||||
/* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
|
/* Writer: upon error, return FFFD (replacement character) */
|
||||||
#define WR(C) if ((C) == -1) { return 0; }
|
#define WR(C) if ((C) == -1) { return 0; }
|
||||||
|
|
||||||
/* Read Unicode character. */
|
/* Read Unicode character. */
|
||||||
|
|||||||
@@ -846,52 +846,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
|||||||
}
|
}
|
||||||
} // while
|
} // while
|
||||||
|
|
||||||
/* --why: print which filter rule decides for this URL, then stop */
|
|
||||||
if (StringNotEmpty(opt->why_url)) {
|
|
||||||
char BIGSTK url[HTS_URLMAXSIZE * 2];
|
|
||||||
lien_adrfil af;
|
|
||||||
|
|
||||||
if (strstr(StringBuff(opt->why_url), ":/") == NULL)
|
|
||||||
snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
|
|
||||||
else
|
|
||||||
strcpybuff(url, StringBuff(opt->why_url));
|
|
||||||
if (ident_url_absolute(url, &af) < 0) {
|
|
||||||
printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
|
|
||||||
} else {
|
|
||||||
char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
|
|
||||||
int jok, jokDepth = 0;
|
|
||||||
|
|
||||||
/* the two forms the wizard tests */
|
|
||||||
strcpybuff(l, jump_identification_const(af.adr));
|
|
||||||
if (*af.fil != '/')
|
|
||||||
strcatbuff(l, "/");
|
|
||||||
strcatbuff(l, af.fil);
|
|
||||||
if (!link_has_authority(af.adr))
|
|
||||||
strcpybuff(lfull, "http://");
|
|
||||||
else
|
|
||||||
lfull[0] = '\0';
|
|
||||||
strcatbuff(lfull, af.adr);
|
|
||||||
if (*af.fil != '/')
|
|
||||||
strcatbuff(lfull, "/");
|
|
||||||
strcatbuff(lfull, af.fil);
|
|
||||||
jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
|
|
||||||
NULL, &jokDepth);
|
|
||||||
if (jok > 0)
|
|
||||||
printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
|
|
||||||
filters[jokDepth]);
|
|
||||||
else if (jok < 0)
|
|
||||||
printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
|
|
||||||
filters[jokDepth]);
|
|
||||||
else
|
|
||||||
printf("%s: no filter rule matches; the wizard decides "
|
|
||||||
"(same-site links are followed by default)" LF,
|
|
||||||
url);
|
|
||||||
}
|
|
||||||
freet(primary);
|
|
||||||
XH_extuninit;
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* load URL file list */
|
/* load URL file list */
|
||||||
/* OPTIMIZED for fast load */
|
/* OPTIMIZED for fast load */
|
||||||
if (StringNotEmpty(opt->filelist)) {
|
if (StringNotEmpty(opt->filelist)) {
|
||||||
|
|||||||
@@ -1803,23 +1803,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
StringCopy(opt->cookies_file, argv[na]);
|
StringCopy(opt->cookies_file, argv[na]);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
|
||||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
|
||||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
|
||||||
printf(
|
|
||||||
"Example: --why \"http://www.example.com/file.zip\"\n");
|
|
||||||
htsmain_free();
|
|
||||||
return -1;
|
|
||||||
} else {
|
|
||||||
na++;
|
|
||||||
if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
|
|
||||||
HTS_PANIC_PRINTF("why URL too long");
|
|
||||||
htsmain_free();
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
StringCopy(opt->why_url, argv[na]);
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
|
case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
|
||||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||||
HTS_PANIC_PRINTF("Option pause needs a blank space and a "
|
HTS_PANIC_PRINTF("Option pause needs a blank space and a "
|
||||||
@@ -1953,20 +1936,18 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
char BIGSTK url[HTS_URLMAXSIZE * 2];
|
char BIGSTK url[HTS_URLMAXSIZE * 2];
|
||||||
char linepos[256];
|
char linepos[256];
|
||||||
int pos;
|
int pos;
|
||||||
LLint cacheNdxLen = 0;
|
char *cacheNdx =
|
||||||
char *cacheNdx = readfile2(
|
readfile(fconcat
|
||||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||||
StringBuff(opt->path_log), "hts-cache/new.ndx"),
|
"hts-cache/new.ndx"));
|
||||||
&cacheNdxLen);
|
|
||||||
cache_init(&cache, opt); /* load cache */
|
cache_init(&cache, opt); /* load cache */
|
||||||
if (cacheNdx != NULL) {
|
if (cacheNdx != NULL) {
|
||||||
char firstline[256];
|
char firstline[256];
|
||||||
char *a = cacheNdx;
|
char *a = cacheNdx;
|
||||||
const char *const end = cacheNdx + cacheNdxLen;
|
|
||||||
|
|
||||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
a += cache_brstr(a, firstline, sizeof(firstline));
|
||||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
a += cache_brstr(a, firstline, sizeof(firstline));
|
||||||
while (a != NULL && a < end) {
|
while(a != NULL) {
|
||||||
a = strchr(a + 1, '\n'); /* start of line */
|
a = strchr(a + 1, '\n'); /* start of line */
|
||||||
if (a) {
|
if (a) {
|
||||||
htsblk r;
|
htsblk r;
|
||||||
@@ -1974,15 +1955,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
/* */
|
/* */
|
||||||
a++;
|
a++;
|
||||||
/* read "host/file" */
|
/* read "host/file" */
|
||||||
a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
|
a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
|
||||||
a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
|
a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
|
||||||
url[0] = '\0';
|
url[0] = '\0';
|
||||||
if (!link_has_authority(afs.af.adr))
|
if (!link_has_authority(afs.af.adr))
|
||||||
strcatbuff(url, "http://");
|
strcatbuff(url, "http://");
|
||||||
strcatbuff(url, afs.af.adr);
|
strcatbuff(url, afs.af.adr);
|
||||||
strcatbuff(url, afs.af.fil);
|
strcatbuff(url, afs.af.fil);
|
||||||
/* read position */
|
/* read position */
|
||||||
a += cache_binput(a, end, linepos, 200);
|
a += binput(a, linepos, 200);
|
||||||
sscanf(linepos, "%d", &pos);
|
sscanf(linepos, "%d", &pos);
|
||||||
if (!hasFilter
|
if (!hasFilter
|
||||||
|| (strjoker(url, filter, NULL, NULL) != NULL)
|
|| (strjoker(url, filter, NULL, NULL) != NULL)
|
||||||
|
|||||||
@@ -94,32 +94,6 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
|
|||||||
return verdict;
|
return verdict;
|
||||||
}
|
}
|
||||||
|
|
||||||
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
|
|
||||||
const char *nom2, LLint *size, int *size_flag,
|
|
||||||
int *depth) {
|
|
||||||
int depth1 = 0, depth2 = 0;
|
|
||||||
int flag1 = 0, flag2 = 0;
|
|
||||||
LLint sz1 = 0, sz2 = 0;
|
|
||||||
int jok1, jok2, use1;
|
|
||||||
|
|
||||||
if (size) {
|
|
||||||
sz1 = *size;
|
|
||||||
sz2 = *size;
|
|
||||||
}
|
|
||||||
jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
|
|
||||||
size_flag ? &flag1 : NULL, &depth1);
|
|
||||||
jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
|
|
||||||
size_flag ? &flag2 : NULL, &depth2);
|
|
||||||
use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
|
|
||||||
if (size)
|
|
||||||
*size = use1 ? sz1 : sz2;
|
|
||||||
if (size_flag)
|
|
||||||
*size_flag = use1 ? flag1 : flag2;
|
|
||||||
if (depth)
|
|
||||||
*depth = use1 ? depth1 : depth2;
|
|
||||||
return use1 ? jok1 : jok2;
|
|
||||||
}
|
|
||||||
|
|
||||||
// supercomparateur joker (tm)
|
// supercomparateur joker (tm)
|
||||||
// compare a et b (b=avec joker dedans), case insensitive [voir CI]
|
// compare a et b (b=avec joker dedans), case insensitive [voir CI]
|
||||||
// renvoi l'adresse de la première lettre de la chaine
|
// renvoi l'adresse de la première lettre de la chaine
|
||||||
@@ -312,7 +286,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
|||||||
if (!unique)
|
if (!unique)
|
||||||
max = (int) strlen(chaine);
|
max = (int) strlen(chaine);
|
||||||
else /* *(a) only match a (not aaaaa) */
|
else /* *(a) only match a (not aaaaa) */
|
||||||
max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
|
max = 1;
|
||||||
while(i < (int) max) {
|
while(i < (int) max) {
|
||||||
if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé
|
if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé
|
||||||
if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
|
if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
|
||||||
|
|||||||
@@ -41,11 +41,6 @@ Please visit our Website: http://www.httrack.com
|
|||||||
|
|
||||||
int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
|
int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
|
||||||
int *size_flag, int *depth);
|
int *size_flag, int *depth);
|
||||||
/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
|
|
||||||
scheme); the match latest in the list wins, a "don't know" verdict defers.
|
|
||||||
Returns the merged verdict; the out-params carry the winner's values. */
|
|
||||||
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
|
|
||||||
const char *nom2, LLint *size, int *size_flag, int *depth);
|
|
||||||
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
|
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
|
||||||
int *size_flag);
|
int *size_flag);
|
||||||
const char *strjokerfind(const char *chaine, const char *joker);
|
const char *strjokerfind(const char *chaine, const char *joker);
|
||||||
|
|||||||
@@ -543,7 +543,6 @@ void help(const char *app, int more) {
|
|||||||
infomsg("Spider options:");
|
infomsg("Spider options:");
|
||||||
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
|
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
|
||||||
infomsg(" %K load extra cookies from a Netscape cookies.txt");
|
infomsg(" %K load extra cookies from a Netscape cookies.txt");
|
||||||
infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit");
|
|
||||||
infomsg
|
infomsg
|
||||||
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
|
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
|
||||||
infomsg
|
infomsg
|
||||||
|
|||||||
16
src/htslib.c
16
src/htslib.c
@@ -1562,7 +1562,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
|||||||
}
|
}
|
||||||
// An empty/whitespace Content-Type value yields no token: keep the
|
// An empty/whitespace Content-Type value yields no token: keep the
|
||||||
// sentinel default rather than reading an uninitialized tempo.
|
// sentinel default rather than reading an uninitialized tempo.
|
||||||
if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
|
if (sscanf(rcvd + p, "%s", tempo) == 1) {
|
||||||
if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
|
if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
|
||||||
strcpybuff(retour->contenttype, tempo);
|
strcpybuff(retour->contenttype, tempo);
|
||||||
else
|
else
|
||||||
@@ -1657,11 +1657,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
|||||||
if (a)
|
if (a)
|
||||||
*a = '\0';
|
*a = '\0';
|
||||||
}
|
}
|
||||||
// Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
|
sscanf(a, "%s", tempo);
|
||||||
if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
|
if (strlen(tempo) < 64) // pas trop long!!
|
||||||
strcpybuff(retour->contentencoding, tempo);
|
strcpybuff(retour->contentencoding, tempo);
|
||||||
else
|
else
|
||||||
retour->contentencoding[0] = '\0';
|
retour->contentencoding[0] = '\0'; // erreur
|
||||||
#if HTS_USEZLIB
|
#if HTS_USEZLIB
|
||||||
/* Check known encodings */
|
/* Check known encodings */
|
||||||
if (retour->contentencoding[0]) {
|
if (retour->contentencoding[0]) {
|
||||||
@@ -2497,10 +2497,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
|
|||||||
// effacer adrfil->adr et adrfil->fil
|
// effacer adrfil->adr et adrfil->fil
|
||||||
adrfil->adr[0] = adrfil->fil[0] = '\0';
|
adrfil->adr[0] = adrfil->fil[0] = '\0';
|
||||||
|
|
||||||
// Reject an over-long URL: a root-relative path is copied whole into fil[].
|
|
||||||
if (strlen(url) >= HTS_URLMAXSIZE * 2)
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
#if HDEBUG
|
#if HDEBUG
|
||||||
printf("protocol: %s\n", url);
|
printf("protocol: %s\n", url);
|
||||||
#endif
|
#endif
|
||||||
@@ -2576,7 +2572,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
|
|||||||
if (*p == '/' || *p == '\\') { /* adrfil->file:///.. */
|
if (*p == '/' || *p == '\\') { /* adrfil->file:///.. */
|
||||||
strcatbuff(adrfil->fil, p); // fichier local ; adrfil->adr="#"
|
strcatbuff(adrfil->fil, p); // fichier local ; adrfil->adr="#"
|
||||||
} else {
|
} else {
|
||||||
if (*p == '\0' || p[1] != ':') { /* empty path: not a DOS drive letter */
|
if (p[1] != ':') {
|
||||||
strcatbuff(adrfil->fil, "//"); /* adrfil->file://server/foo */
|
strcatbuff(adrfil->fil, "//"); /* adrfil->file://server/foo */
|
||||||
strcatbuff(adrfil->fil, p);
|
strcatbuff(adrfil->fil, p);
|
||||||
} else {
|
} else {
|
||||||
@@ -6006,7 +6002,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
|||||||
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
|
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
|
||||||
StringCopy(opt->strip_query, "");
|
StringCopy(opt->strip_query, "");
|
||||||
StringCopy(opt->cookies_file, "");
|
StringCopy(opt->cookies_file, "");
|
||||||
StringCopy(opt->why_url, "");
|
|
||||||
opt->pause_min_ms = 0;
|
opt->pause_min_ms = 0;
|
||||||
opt->pause_max_ms = 0;
|
opt->pause_max_ms = 0;
|
||||||
opt->ftp_proxy = HTS_TRUE;
|
opt->ftp_proxy = HTS_TRUE;
|
||||||
@@ -6154,7 +6149,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
|
|||||||
StringFree(opt->mod_blacklist);
|
StringFree(opt->mod_blacklist);
|
||||||
StringFree(opt->strip_query);
|
StringFree(opt->strip_query);
|
||||||
StringFree(opt->cookies_file);
|
StringFree(opt->cookies_file);
|
||||||
StringFree(opt->why_url);
|
|
||||||
|
|
||||||
StringFree(opt->path_html);
|
StringFree(opt->path_html);
|
||||||
StringFree(opt->path_html_utf8);
|
StringFree(opt->path_html_utf8);
|
||||||
|
|||||||
@@ -536,8 +536,6 @@ struct httrackp {
|
|||||||
(--cookies-file) */
|
(--cookies-file) */
|
||||||
int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
|
int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
|
||||||
int pause_max_ms; /**< inter-file pause upper bound, ms */
|
int pause_max_ms; /**< inter-file pause upper bound, ms */
|
||||||
String why_url; /**< URL to diagnose (--why): print the deciding filter rule
|
|
||||||
and exit without crawling */
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Running statistics for a mirror. */
|
/* Running statistics for a mirror. */
|
||||||
|
|||||||
@@ -46,7 +46,6 @@ Please visit our Website: http://www.httrack.com
|
|||||||
#include "htsdefines.h"
|
#include "htsdefines.h"
|
||||||
#include "htslib.h"
|
#include "htslib.h"
|
||||||
#include "htsparse.h"
|
#include "htsparse.h"
|
||||||
#include "htscache.h"
|
|
||||||
#include "htscache_selftest.h"
|
#include "htscache_selftest.h"
|
||||||
#include "htsdns_selftest.h"
|
#include "htsdns_selftest.h"
|
||||||
#include "htscharset.h"
|
#include "htscharset.h"
|
||||||
@@ -580,16 +579,15 @@ static int st_filter(httrackp *opt, int argc, char **argv) {
|
|||||||
int matched;
|
int matched;
|
||||||
|
|
||||||
(void) opt;
|
(void) opt;
|
||||||
if (argc < 1) {
|
if (argc < 2) {
|
||||||
fprintf(stderr, "filter: needs a filter pattern and a string\n");
|
fprintf(stderr, "filter: needs a filter pattern and a string\n");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
/* exact-size heap copies so a sanitizer traps an over-read; a missing
|
/* exact-size heap copies so a sanitizer traps any over-read of the pattern */
|
||||||
subject means "" (not reachable as a CLI arg) */
|
str = strdupt(argv[1]);
|
||||||
str = strdupt(argc >= 2 ? argv[1] : "");
|
|
||||||
pat = strdupt(argv[0]);
|
pat = strdupt(argv[0]);
|
||||||
matched = strjoker(str, pat, NULL, NULL) != NULL;
|
matched = strjoker(str, pat, NULL, NULL) != NULL;
|
||||||
printf("%s does %s %s\n", str, matched ? "match" : "NOT match", argv[0]);
|
printf("%s does %s %s\n", argv[1], matched ? "match" : "NOT match", argv[0]);
|
||||||
freet(str);
|
freet(str);
|
||||||
freet(pat);
|
freet(pat);
|
||||||
return 0;
|
return 0;
|
||||||
@@ -621,26 +619,6 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
|
|
||||||
static int st_filterdual(httrackp *opt, int argc, char **argv) {
|
|
||||||
int depth = -1, verdict;
|
|
||||||
|
|
||||||
(void) opt;
|
|
||||||
if (argc < 3) {
|
|
||||||
fprintf(stderr,
|
|
||||||
"filterdual: needs <string1> <string2> <filter> [filter...]\n");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
|
|
||||||
NULL, &depth);
|
|
||||||
printf("verdict=%s rule=%d\n",
|
|
||||||
verdict > 0 ? "allowed"
|
|
||||||
: verdict < 0 ? "forbidden"
|
|
||||||
: "unknown",
|
|
||||||
depth);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int st_simplify(httrackp *opt, int argc, char **argv) {
|
static int st_simplify(httrackp *opt, int argc, char **argv) {
|
||||||
(void) opt;
|
(void) opt;
|
||||||
if (argc < 1) {
|
if (argc < 1) {
|
||||||
@@ -1116,33 +1094,6 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Split a URL into (adr, fil), or print "error" if rejected. A second arg pads
|
|
||||||
the URL with that many 'a's to reach lengths a CLI arg can't. */
|
|
||||||
static int st_identurl(httrackp *opt, int argc, char **argv) {
|
|
||||||
lien_adrfil af;
|
|
||||||
char *url;
|
|
||||||
size_t len, pad = 0;
|
|
||||||
|
|
||||||
(void) opt;
|
|
||||||
if (argc < 1) {
|
|
||||||
fprintf(stderr, "identurl: needs a URL\n");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
if (argc >= 2)
|
|
||||||
pad = (size_t) atoi(argv[1]);
|
|
||||||
len = strlen(argv[0]);
|
|
||||||
url = malloct(len + pad + 1);
|
|
||||||
memcpy(url, argv[0], len);
|
|
||||||
memset(url + len, 'a', pad);
|
|
||||||
url[len + pad] = '\0';
|
|
||||||
if (ident_url_absolute(url, &af) >= 0)
|
|
||||||
printf("adr=%s fil=%s\n", af.adr, af.fil);
|
|
||||||
else
|
|
||||||
printf("error\n");
|
|
||||||
freet(url);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
|
/* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
|
||||||
no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
|
no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
|
||||||
registering an already-crawled link (dedup/collision paths). */
|
registering an already-crawled link (dedup/collision paths). */
|
||||||
@@ -1164,26 +1115,6 @@ static int st_header(httrackp *opt, int argc, char **argv) {
|
|||||||
treathead(NULL, "www.example.com", "/", &r, line);
|
treathead(NULL, "www.example.com", "/", &r, line);
|
||||||
}
|
}
|
||||||
printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
|
printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
|
||||||
printf("contentencoding=%s\n", r.contentencoding);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* An over-long header value must not overflow treathead's tempo[1100]. */
|
|
||||||
static int st_headerlong(httrackp *opt, int argc, char **argv) {
|
|
||||||
htsblk r;
|
|
||||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
|
||||||
const char *const name = argc >= 1 ? argv[0] : "Content-Type:";
|
|
||||||
const int pad = 1500; /* > tempo[1100] */
|
|
||||||
size_t n;
|
|
||||||
|
|
||||||
(void) opt;
|
|
||||||
memset(&r, 0, sizeof(r));
|
|
||||||
n = (size_t) snprintf(line, sizeof(line), "%s ", name);
|
|
||||||
memset(line + n, 'a', pad);
|
|
||||||
line[n + pad] = '\0';
|
|
||||||
treathead(NULL, "www.example.com", "/", &r, line);
|
|
||||||
printf("contenttype_len=%d contentencoding_len=%d\n",
|
|
||||||
(int) strlen(r.contenttype), (int) strlen(r.contentencoding));
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1391,78 +1322,6 @@ static int st_cache(httrackp *opt, int argc, char **argv) {
|
|||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A corrupt cache index (.ndx) must not walk the length-prefixed scan past
|
|
||||||
the buffer. Checks the two primitives the loader is built from. */
|
|
||||||
static int st_cacheindex(httrackp *opt, int argc, char **argv) {
|
|
||||||
int fail = 0;
|
|
||||||
|
|
||||||
(void) opt;
|
|
||||||
(void) argc;
|
|
||||||
(void) argv;
|
|
||||||
|
|
||||||
/* A length prefix that overstates the bytes present must bound the advance
|
|
||||||
to the buffer, not trust the declared length. */
|
|
||||||
{
|
|
||||||
static const char src[] = "32768\nCACHE-1.1";
|
|
||||||
const size_t len = sizeof(src) - 1;
|
|
||||||
char *buf = malloct(len + 1);
|
|
||||||
char s[256];
|
|
||||||
int off;
|
|
||||||
|
|
||||||
memcpy(buf, src, len + 1);
|
|
||||||
off = cache_brstr(buf, s, sizeof(s));
|
|
||||||
if (off > (int) len) {
|
|
||||||
printf("cacheindex: over-advance off=%d len=%d\n", off, (int) len);
|
|
||||||
fail = 1;
|
|
||||||
}
|
|
||||||
if (strcmp(s, "CACHE-1.1") != 0) {
|
|
||||||
printf("cacheindex: value=%s\n", s);
|
|
||||||
fail = 1;
|
|
||||||
}
|
|
||||||
freet(buf);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* cache_binput reads a field while in bounds, but refuses one starting at
|
|
||||||
or past end-of-buffer. */
|
|
||||||
{
|
|
||||||
char buf[8] = "ab\ncd";
|
|
||||||
const char *const end = buf + 5;
|
|
||||||
char s[16];
|
|
||||||
|
|
||||||
if (cache_binput(buf, end, s, sizeof(s)) != 3 || strcmp(s, "ab") != 0)
|
|
||||||
fail = 1; /* normal read: "ab" then the '\n', 3 bytes consumed */
|
|
||||||
if (cache_binput(end, end, s, sizeof(s)) != 0 || s[0] != '\0')
|
|
||||||
fail = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Drive the full loader scan over a truncated index: a declared length that
|
|
||||||
overshoots plus a half-written entry. ASan aborts here on the pre-fix
|
|
||||||
scan; the cursor must never leave the buffer. */
|
|
||||||
{
|
|
||||||
static const char src[] = "9\nCACHE-1.1\n99\nwww.example.com\n/a";
|
|
||||||
const size_t len = sizeof(src) - 1;
|
|
||||||
char *buf = malloct(len + 1);
|
|
||||||
const char *const end = buf + len;
|
|
||||||
char line[256];
|
|
||||||
char *a = buf;
|
|
||||||
|
|
||||||
memcpy(buf, src, len + 1);
|
|
||||||
a += cache_brstr(a, line, sizeof(line));
|
|
||||||
a += cache_brstr(a, line, sizeof(line));
|
|
||||||
while (a != NULL && a < end) {
|
|
||||||
a = strchr(a + 1, '\n');
|
|
||||||
if (a == NULL)
|
|
||||||
break;
|
|
||||||
a++;
|
|
||||||
a += cache_binput(a, end, line, sizeof(line));
|
|
||||||
}
|
|
||||||
freet(buf);
|
|
||||||
}
|
|
||||||
|
|
||||||
printf("cacheindex: %s\n", fail ? "FAIL" : "OK");
|
|
||||||
return fail;
|
|
||||||
}
|
|
||||||
|
|
||||||
static int st_cache_golden(httrackp *opt, int argc, char **argv) {
|
static int st_cache_golden(httrackp *opt, int argc, char **argv) {
|
||||||
int regen, err;
|
int regen, err;
|
||||||
|
|
||||||
@@ -2247,8 +2106,6 @@ static const struct selftest_entry {
|
|||||||
{"filtersize", "<size> <string> <filter>...",
|
{"filtersize", "<size> <string> <filter>...",
|
||||||
"size-aware filter verdict (negative size = unknown/scan time)",
|
"size-aware filter verdict (negative size = unknown/scan time)",
|
||||||
st_filtersize},
|
st_filtersize},
|
||||||
{"filterdual", "<string1> <string2> <filter>...",
|
|
||||||
"merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
|
|
||||||
{"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
|
{"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
|
||||||
{"stripquery", "", "--strip-query pattern/key stripping self-test",
|
{"stripquery", "", "--strip-query pattern/key stripping self-test",
|
||||||
st_stripquery},
|
st_stripquery},
|
||||||
@@ -2275,19 +2132,13 @@ static const struct selftest_entry {
|
|||||||
st_relative},
|
st_relative},
|
||||||
{"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
|
{"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
|
||||||
st_resolve},
|
st_resolve},
|
||||||
{"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
|
|
||||||
{"header", "<raw-header-line> ...", "response header-line parsing",
|
{"header", "<raw-header-line> ...", "response header-line parsing",
|
||||||
st_header},
|
st_header},
|
||||||
{"headerlong", "[header-name:]",
|
|
||||||
"over-long header value must not overflow the parse scratch",
|
|
||||||
st_headerlong},
|
|
||||||
{"savename", "<fil> <content-type> [key=value ...]",
|
{"savename", "<fil> <content-type> [key=value ...]",
|
||||||
"local save-name for a URL", st_savename},
|
"local save-name for a URL", st_savename},
|
||||||
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
|
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
|
||||||
st_sniff},
|
st_sniff},
|
||||||
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
|
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
|
||||||
{"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
|
|
||||||
st_cacheindex},
|
|
||||||
{"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
|
{"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
|
||||||
st_cache_golden},
|
st_cache_golden},
|
||||||
{"cache-writefail", "<dir>", "cache write-failure handling self-test",
|
{"cache-writefail", "<dir>", "cache write-failure handling self-test",
|
||||||
|
|||||||
@@ -524,11 +524,28 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
|||||||
|
|
||||||
// filters, 0=sait pas 1=ok -1=interdit
|
// filters, 0=sait pas 1=ok -1=interdit
|
||||||
{
|
{
|
||||||
int jokDepth = 0;
|
int jokDepth1 = 0, jokDepth2 = 0;
|
||||||
|
int jok1 = 0, jok2 = 0;
|
||||||
|
|
||||||
jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
|
jok1 =
|
||||||
NULL, NULL, &jokDepth);
|
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
|
||||||
mdepth = _FILTERS[jokDepth];
|
&jokDepth1);
|
||||||
|
jok2 =
|
||||||
|
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
|
||||||
|
&jokDepth2);
|
||||||
|
if (jok2 == 0) { // #2 doesn't know
|
||||||
|
jok = jok1; // then, use #1
|
||||||
|
mdepth = _FILTERS[jokDepth1];
|
||||||
|
} else if (jok1 == 0) { // #1 doesn't know
|
||||||
|
jok = jok2; // then, use #2
|
||||||
|
mdepth = _FILTERS[jokDepth2];
|
||||||
|
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
|
||||||
|
jok = jok1;
|
||||||
|
mdepth = _FILTERS[jokDepth1];
|
||||||
|
} else { // #2 matching rule is "after" #1, then it is prioritary
|
||||||
|
jok = jok2;
|
||||||
|
mdepth = _FILTERS[jokDepth2];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (jok == 1) { // autorisé
|
if (jok == 1) { // autorisé
|
||||||
@@ -948,10 +965,34 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
|
|||||||
|
|
||||||
// filters, 0=sait pas 1=ok -1=interdit
|
// filters, 0=sait pas 1=ok -1=interdit
|
||||||
{
|
{
|
||||||
sz = size;
|
int jokDepth1 = 0, jokDepth2 = 0;
|
||||||
jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
|
int jok1 = 0, jok2 = 0;
|
||||||
*opt->filters.filptr, lfull, l, &sz, &size_flag,
|
LLint sz1 = size, sz2 = size;
|
||||||
NULL);
|
int size_flag1 = 0, size_flag2 = 0;
|
||||||
|
|
||||||
|
jok1 =
|
||||||
|
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
|
||||||
|
lfull, &sz1, &size_flag1, &jokDepth1);
|
||||||
|
jok2 =
|
||||||
|
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
|
||||||
|
l, &sz2, &size_flag2, &jokDepth2);
|
||||||
|
if (jok2 == 0) { // #2 doesn't know
|
||||||
|
jok = jok1; // then, use #1
|
||||||
|
sz = sz1;
|
||||||
|
size_flag = size_flag1;
|
||||||
|
} else if (jok1 == 0) { // #1 doesn't know
|
||||||
|
jok = jok2; // then, use #2
|
||||||
|
sz = sz2;
|
||||||
|
size_flag = size_flag2;
|
||||||
|
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
|
||||||
|
jok = jok1;
|
||||||
|
sz = sz1;
|
||||||
|
size_flag = size_flag1;
|
||||||
|
} else { // #2 matching rule is "after" #1, then it is prioritary
|
||||||
|
jok = jok2;
|
||||||
|
sz = sz2;
|
||||||
|
size_flag = size_flag2;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// log
|
// log
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# Cache-index (.ndx) parse must stay inside the buffer on a corrupt length
|
|
||||||
# prefix (ASan aborts here on the pre-fix binary; the OK check gates the
|
|
||||||
# non-sanitized build too).
|
|
||||||
|
|
||||||
test "$(httrack -O /dev/null -#test=cacheindex)" == 'cacheindex: OK'
|
|
||||||
@@ -12,11 +12,6 @@ match() {
|
|||||||
nomatch() {
|
nomatch() {
|
||||||
test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does NOT match $1" || exit 1
|
test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does NOT match $1" || exit 1
|
||||||
}
|
}
|
||||||
# single-arg call means an empty subject (unreachable as a CLI arg); '*(' used
|
|
||||||
# to force max=1 and over-read past it.
|
|
||||||
nomatch_empty() {
|
|
||||||
test "$(httrack -O /dev/null -#test=filter "$1")" == " does NOT match $1" || exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# bare star matches everything
|
# bare star matches everything
|
||||||
match '*' 'anything/at/all'
|
match '*' 'anything/at/all'
|
||||||
@@ -127,7 +122,3 @@ nomatch '*[file]end' 'foo?xend'
|
|||||||
nomatch '*[name]X' 'abc?X'
|
nomatch '*[name]X' 'abc?X'
|
||||||
match '*[file]' 'foo?x=1' # trailing query: tolerated, as for *.aspx
|
match '*[file]' 'foo?x=1' # trailing query: tolerated, as for *.aspx
|
||||||
match '*.aspx' 'page.aspx?y=2'
|
match '*.aspx' 'page.aspx?y=2'
|
||||||
|
|
||||||
# empty subject against a unique-match allow-all pattern (heap over-read regress)
|
|
||||||
nomatch_empty '*(('
|
|
||||||
nomatch_empty '**(('
|
|
||||||
|
|||||||
@@ -1,20 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
|
|
||||||
# the match latest in the filter list wins, an unknown verdict defers.
|
|
||||||
|
|
||||||
fdual() {
|
|
||||||
local want="$1"
|
|
||||||
shift
|
|
||||||
test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
fdual 'verdict=unknown rule=0' zzz yyy '+a*' # neither form matches
|
|
||||||
fdual 'verdict=forbidden rule=0' aaa zzz '-a*' # only form 1 matches
|
|
||||||
fdual 'verdict=forbidden rule=0' zzz aaa '-a*' # only form 2 matches
|
|
||||||
fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
|
|
||||||
fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
|
|
||||||
fdual 'verdict=allowed rule=0' a aa '+a*' # tie: form 1 wins
|
|
||||||
@@ -27,16 +27,3 @@ hdr 'contenttype= cdispo=report.pdf' \
|
|||||||
# Path components in the filename are dropped on the wire (RFC 2616).
|
# Path components in the filename are dropped on the wire (RFC 2616).
|
||||||
hdr 'contenttype= cdispo=evil.pdf' \
|
hdr 'contenttype= cdispo=evil.pdf' \
|
||||||
'Content-Disposition: attachment; filename="../../evil.pdf"'
|
'Content-Disposition: attachment; filename="../../evil.pdf"'
|
||||||
|
|
||||||
# An over-long header value must not overflow the parse scratch (ASan aborts
|
|
||||||
# here on the pre-fix binary).
|
|
||||||
test "$(httrack -O /dev/null -#test=headerlong 'Content-Type:')" \
|
|
||||||
== 'contenttype_len=32 contentencoding_len=0'
|
|
||||||
test "$(httrack -O /dev/null -#test=headerlong 'Content-Encoding:')" \
|
|
||||||
== 'contenttype_len=0 contentencoding_len=0'
|
|
||||||
|
|
||||||
# An empty Content-Encoding yields no token: leave it empty, don't read the
|
|
||||||
# uninitialized scratch (MSan aborts here on the pre-fix binary).
|
|
||||||
enc() { httrack -O /dev/null -#test=header "$1" | grep '^contentencoding='; }
|
|
||||||
test "$(enc 'Content-Encoding:')" == 'contentencoding='
|
|
||||||
test "$(enc 'Content-Encoding: gzip')" == 'contentencoding=gzip'
|
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# ident_url_absolute splits a raw URL into (adr, fil), the first parser to touch
|
|
||||||
# it. -#test=identurl <url> [pad] prints "adr=.. fil=.." or "error"; pad appends
|
|
||||||
# N 'a's, reaching lengths a CLI arg can't.
|
|
||||||
|
|
||||||
split() {
|
|
||||||
test "$(httrack -O /dev/null -#test=identurl "$1")" == "$2" || exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
split 'http://www.example.com/a/b/../c.html' 'adr=www.example.com fil=/a/c.html'
|
|
||||||
split 'ftp://ftp.example.com/pub/file.txt' 'adr=ftp://ftp.example.com fil=/pub/file.txt'
|
|
||||||
split 'file:///etc/hostname' 'adr=file:// fil=/etc/hostname'
|
|
||||||
|
|
||||||
# empty-path file:// used to read one byte past the URL (p[1] with *p=0)
|
|
||||||
split 'file://' 'adr=file:// fil=//'
|
|
||||||
|
|
||||||
# a root-relative path is copied whole into fil[HTS_URLMAXSIZE*2], so a URL at
|
|
||||||
# the buffer size is the tightest overflow; it must be rejected, not abort.
|
|
||||||
split_pad() {
|
|
||||||
test "$(httrack -O /dev/null -#test=identurl "$1" "$2")" == "$3" || exit 1
|
|
||||||
}
|
|
||||||
split_pad '/' 2047 'error' # 1 + 2047 = 2048 = HTS_URLMAXSIZE*2
|
|
||||||
split_pad 'http://h/' 3000 'error'
|
|
||||||
@@ -8,8 +8,8 @@ set -euo pipefail
|
|||||||
|
|
||||||
enc() { test "$(httrack -O /dev/null -#test=idna-encode "$1")" == "$2" || exit 1; }
|
enc() { test "$(httrack -O /dev/null -#test=idna-encode "$1")" == "$2" || exit 1; }
|
||||||
dec() { test "$(httrack -O /dev/null -#test=idna-decode "$1")" == "$2" || exit 1; }
|
dec() { test "$(httrack -O /dev/null -#test=idna-decode "$1")" == "$2" || exit 1; }
|
||||||
# a malformed encode input must be rejected (empty output), not encoded or leaked.
|
# crash probe: malformed ACE input must exit cleanly, not abort.
|
||||||
enc_bad() { test -z "$(httrack -O /dev/null -#test=idna-encode "$1" 2>/dev/null)" || exit 1; }
|
runs() { httrack -O /dev/null -#test=idna-decode "$1" >/dev/null 2>&1 || exit 1; }
|
||||||
|
|
||||||
# encode
|
# encode
|
||||||
enc 'www.café.com' 'www.xn--caf-dma.com'
|
enc 'www.café.com' 'www.xn--caf-dma.com'
|
||||||
@@ -33,9 +33,6 @@ enc 'xn--already-encoded.com' 'xn--already-encoded.com'
|
|||||||
# an empty punycode payload decodes back to the bare xn-- label
|
# an empty punycode payload decodes back to the bare xn-- label
|
||||||
dec 'xn--' 'xn--'
|
dec 'xn--' 'xn--'
|
||||||
|
|
||||||
# malformed ACE payloads decode to a stable (garbage) string, not a crash
|
# malformed ACE payloads (invalid base-36, garbage) must not crash
|
||||||
dec 'xn--!!!' '㚯'
|
runs 'xn--!!!'
|
||||||
dec 'xn--already-encoded.com' 'ǮalǭǮreaǭǫdy.com'
|
runs 'xn--already-encoded.com'
|
||||||
|
|
||||||
# second label has a truncated UTF-8 sequence: used to leak the segment buffer.
|
|
||||||
enc_bad "$(printf 'b\xc3\xbcchev.\xe4\xbe\x8b\xe5\xad\x62\xc3\xbccheple')"
|
|
||||||
|
|||||||
@@ -1,35 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# --why: report which +/- filter rule decides for a URL, without crawling.
|
|
||||||
|
|
||||||
tmpdir="$(mktemp -d)"
|
|
||||||
trap 'rm -rf "$tmpdir"' EXIT
|
|
||||||
|
|
||||||
why() {
|
|
||||||
local want="$1" out
|
|
||||||
shift
|
|
||||||
out="$(httrack -O "$tmpdir/p" -q "$@" |
|
|
||||||
grep -E 'accepted|rejected|no filter|unable')"
|
|
||||||
test "$out" == "$want" || {
|
|
||||||
echo "want: $want"
|
|
||||||
echo "got : $out"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
base=http://www.example.com
|
|
||||||
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
|
|
||||||
--why "$base/x.zip" "$base/" '+*.png' '-*.zip'
|
|
||||||
why "$base/a.png: accepted by rule #1 (+*.png)" \
|
|
||||||
--why "$base/a.png" "$base/" '+*.png' '-*.zip'
|
|
||||||
why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
|
|
||||||
--why "$base/i.html" "$base/" '+*.png' '-*.zip'
|
|
||||||
# scheme-less input gets the same http:// default as primary URLs
|
|
||||||
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
|
|
||||||
--why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
|
|
||||||
# the rule latest in the list decides
|
|
||||||
why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
|
|
||||||
--why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'
|
|
||||||
@@ -35,14 +35,11 @@ TESTS = \
|
|||||||
01_engine-entities.test \
|
01_engine-entities.test \
|
||||||
01_engine-filelist.test \
|
01_engine-filelist.test \
|
||||||
01_engine-filter.test \
|
01_engine-filter.test \
|
||||||
01_engine-filterdual.test \
|
|
||||||
01_engine-ftp-line.test \
|
01_engine-ftp-line.test \
|
||||||
01_engine-ftp-userpass.test \
|
01_engine-ftp-userpass.test \
|
||||||
01_engine-cacheindex.test \
|
|
||||||
01_engine-hashtable.test \
|
01_engine-hashtable.test \
|
||||||
01_engine-header.test \
|
01_engine-header.test \
|
||||||
01_engine-idna.test \
|
01_engine-idna.test \
|
||||||
01_engine-identurl.test \
|
|
||||||
01_engine-escape-room.test \
|
01_engine-escape-room.test \
|
||||||
01_engine-inplace-escape.test \
|
01_engine-inplace-escape.test \
|
||||||
01_engine-java.test \
|
01_engine-java.test \
|
||||||
@@ -107,7 +104,6 @@ TESTS = \
|
|||||||
36_local-bigcrawl.test \
|
36_local-bigcrawl.test \
|
||||||
37_local-cache-outage.test \
|
37_local-cache-outage.test \
|
||||||
38_local-update-304.test \
|
38_local-update-304.test \
|
||||||
39_local-delayed-cancel.test \
|
39_local-delayed-cancel.test
|
||||||
40_local-why.test
|
|
||||||
|
|
||||||
CLEANFILES = check-network_sh.cache
|
CLEANFILES = check-network_sh.cache
|
||||||
|
|||||||
Reference in New Issue
Block a user