mirror of
https://github.com/xroche/httrack.git
synced 2026-07-08 18:06:31 +03:00
Compare commits
2 Commits
px-1-why
...
remove-dea
| Author | SHA1 | Date | |
|---|---|---|---|
|
cf483a24e4 | ||
|
|
abaea566bd |
39
.github/workflows/ci.yml
vendored
39
.github/workflows/ci.yml
vendored
@@ -278,45 +278,6 @@ jobs:
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
|
||||
# parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
|
||||
# (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
|
||||
# catastrophic backtracking -- and pins the regression seeds for the bugs the
|
||||
# fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
|
||||
fuzz:
|
||||
name: fuzz (libFuzzer corpus replay, clang)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential clang autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev
|
||||
|
||||
- name: Configure (fuzzers, static)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure CC=clang \
|
||||
CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
|
||||
LDFLAGS="-fsanitize=address,undefined" \
|
||||
--enable-fuzzers --disable-shared
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Replay corpora
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
run: bash fuzz/run-fuzzers.sh fuzz check
|
||||
|
||||
# Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
|
||||
# the configuration users on minimal systems build, and one libssl is not even
|
||||
# installed here so configure cannot silently re-enable it. The matrix above
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
|
||||
SUBDIRS = src man m4 libtest templates lang html tests fuzz
|
||||
SUBDIRS = src man m4 libtest templates lang html tests
|
||||
|
||||
ACLOCAL_AMFLAGS = -I m4
|
||||
|
||||
|
||||
32
configure.ac
32
configure.ac
@@ -310,37 +310,6 @@ AC_ARG_ENABLE([online-unit-tests],
|
||||
])
|
||||
AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
|
||||
|
||||
## libFuzzer harnesses (fuzz/); requires clang
|
||||
AC_MSG_CHECKING(whether to build fuzzers)
|
||||
AC_ARG_ENABLE([fuzzers],
|
||||
[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
|
||||
[
|
||||
case "${enableval}" in
|
||||
no|yes)
|
||||
fuzzers=$enableval
|
||||
AC_MSG_RESULT($enableval)
|
||||
;;
|
||||
*)
|
||||
AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
|
||||
;;
|
||||
esac
|
||||
],
|
||||
[
|
||||
fuzzers=no
|
||||
AC_MSG_RESULT(no)
|
||||
])
|
||||
if test x"$fuzzers" = x"yes"; then
|
||||
# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
|
||||
AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
|
||||
[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
|
||||
[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
|
||||
# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
|
||||
if test x"$enable_shared" != x"no"; then
|
||||
AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
|
||||
fi
|
||||
fi
|
||||
AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
|
||||
|
||||
# Final output
|
||||
AC_CONFIG_FILES([
|
||||
Makefile
|
||||
@@ -352,6 +321,5 @@ lang/Makefile
|
||||
html/Makefile
|
||||
libtest/Makefile
|
||||
tests/Makefile
|
||||
fuzz/Makefile
|
||||
])
|
||||
AC_OUTPUT
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
|
||||
if FUZZERS
|
||||
noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
|
||||
fuzz-unescape fuzz-filters fuzz-url
|
||||
endif
|
||||
|
||||
AM_CPPFLAGS = \
|
||||
@DEFAULT_CFLAGS@ \
|
||||
@THREADS_CFLAGS@ \
|
||||
@V6_FLAG@ \
|
||||
@LFS_FLAG@ \
|
||||
-I$(top_srcdir)/src \
|
||||
-I$(top_srcdir)/src/coucal
|
||||
|
||||
# Static-link libhttrack.la: the internal symbols are hidden in the .so.
|
||||
AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
|
||||
LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
|
||||
|
||||
fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
|
||||
fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
|
||||
fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
|
||||
fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
|
||||
fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
|
||||
fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
|
||||
fuzz_url_SOURCES = fuzz-url.c fuzz.h
|
||||
|
||||
# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
|
||||
EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
|
||||
corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
|
||||
corpus/idna/idna.txt corpus/idna/unicode.txt \
|
||||
corpus/idna/regress-multilabel-leak.txt \
|
||||
corpus/entities/entities.txt \
|
||||
corpus/unescape/percent.txt \
|
||||
corpus/filters/filter.bin corpus/filters/filter-size.bin \
|
||||
corpus/filters/regress-empty-subject-unique.bin \
|
||||
corpus/url/http-url.txt corpus/url/relative-path.txt \
|
||||
corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt
|
||||
@@ -1,15 +0,0 @@
|
||||
# Fuzzing httrack
|
||||
|
||||
libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
|
||||
|
||||
```sh
|
||||
./bootstrap
|
||||
mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
|
||||
CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
|
||||
LDFLAGS="-fsanitize=address,undefined" \
|
||||
bash /path/to/httrack/configure --enable-fuzzers --disable-shared
|
||||
make
|
||||
bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60 # 60s per target
|
||||
```
|
||||
|
||||
Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.
|
||||
@@ -1 +0,0 @@
|
||||
café naďve ¤
|
||||
@@ -1 +0,0 @@
|
||||
コンピュータ
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
&<>A☃é¬arealentity;�
|
||||
Binary file not shown.
Binary file not shown.
@@ -1 +0,0 @@
|
||||
**((
|
||||
@@ -1 +0,0 @@
|
||||
xn--bcher-kva.example.com
|
||||
@@ -1 +0,0 @@
|
||||
büchev.例åbücheple
|
||||
@@ -1 +0,0 @@
|
||||
bücher.例子.example
|
||||
@@ -1 +0,0 @@
|
||||
<html><head><meta charset="utf-8"></head><body>x</body></html>
|
||||
@@ -1 +0,0 @@
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
@@ -1 +0,0 @@
|
||||
%41%zz%%20%c3%a9+%2e%2e%2f
|
||||
@@ -1 +0,0 @@
|
||||
http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag
|
||||
@@ -1 +0,0 @@
|
||||
file://
|
||||
@@ -1 +0,0 @@
|
||||
ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../
|
||||
@@ -1 +0,0 @@
|
||||
ftp://ftp.example.com/pub/../file.txt
|
||||
@@ -1,84 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
|
||||
UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
|
||||
#include "fuzz.h"
|
||||
#include "htscharset.h"
|
||||
|
||||
static const char *const charsets[] = {
|
||||
"utf-8", "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
|
||||
"us-ascii", "shift_jis", "euc-jp", "iso-2022-jp", "gb2312",
|
||||
"big5", "euc-kr", "koi8-r", "utf-16", "unknown-charset",
|
||||
};
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
const char *charset;
|
||||
char *s;
|
||||
|
||||
if (size == 0)
|
||||
return 0;
|
||||
charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
|
||||
data++, size--;
|
||||
s = fuzz_strdup(data, size);
|
||||
|
||||
{
|
||||
char *utf8 = hts_convertStringToUTF8(s, size, charset);
|
||||
freet(utf8);
|
||||
}
|
||||
{
|
||||
char *enc = hts_convertStringFromUTF8(s, size, charset);
|
||||
freet(enc);
|
||||
}
|
||||
{
|
||||
size_t nChars = 0;
|
||||
hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
|
||||
|
||||
if (ucs != NULL) {
|
||||
char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
|
||||
freet(back);
|
||||
freet(ucs);
|
||||
}
|
||||
}
|
||||
{
|
||||
size_t i = 0;
|
||||
|
||||
while (i < size) {
|
||||
hts_UCS4 uc = 0;
|
||||
const size_t nr = hts_readUTF8(s + i, size - i, &uc);
|
||||
char out[8];
|
||||
|
||||
if (nr == 0)
|
||||
break;
|
||||
hts_writeUTF8(uc, out, sizeof(out));
|
||||
i += nr;
|
||||
}
|
||||
}
|
||||
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
|
||||
destination size, so truncation bounds get exercised too. */
|
||||
#include "fuzz.h"
|
||||
#include "htsencoding.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static const size_t dsizes[] = {1, 2, 8, 64, 4096};
|
||||
size_t dsize;
|
||||
char *src, *dest;
|
||||
|
||||
if (size == 0)
|
||||
return 0;
|
||||
dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
|
||||
data++, size--;
|
||||
src = fuzz_strdup(data, size);
|
||||
dest = malloct(dsize);
|
||||
|
||||
(void) hts_unescapeEntities(src, dest, dsize);
|
||||
(void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
|
||||
|
||||
freet(dest);
|
||||
freet(src);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
|
||||
here). Input splits on the first NUL: pattern, then subject string. */
|
||||
#include "fuzz.h"
|
||||
#include "htsfilters.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *buf = fuzz_strdup(data, size);
|
||||
const char *joker = buf;
|
||||
const uint8_t *sep = memchr(data, '\0', size);
|
||||
/* subject in its own allocation so ASan bounds it apart from the pattern */
|
||||
char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
|
||||
: fuzz_strdup(data + size, 0);
|
||||
|
||||
(void) strjoker(nom, joker, NULL, NULL);
|
||||
{
|
||||
LLint sz = (LLint) size;
|
||||
int size_flag = 0;
|
||||
|
||||
(void) strjoker(nom, joker, &sz, &size_flag);
|
||||
}
|
||||
(void) strjokerfind(nom, joker);
|
||||
{
|
||||
char *filter = malloct(strlen(joker) + 2);
|
||||
char *filters[1];
|
||||
LLint sz = (LLint) size;
|
||||
int size_flag = 0, depth = 0;
|
||||
|
||||
filter[0] = '-';
|
||||
memcpy(filter + 1, joker, strlen(joker) + 1);
|
||||
filters[0] = filter;
|
||||
(void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
|
||||
freet(filter);
|
||||
}
|
||||
|
||||
freet(nom);
|
||||
freet(buf);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
|
||||
#include "fuzz.h"
|
||||
#include "htscharset.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *s = fuzz_strdup(data, size);
|
||||
|
||||
{
|
||||
char *idna = hts_convertStringUTF8ToIDNA(s, size);
|
||||
freet(idna);
|
||||
}
|
||||
{
|
||||
char *utf8 = hts_convertStringIDNAToUTF8(s, size);
|
||||
freet(utf8);
|
||||
}
|
||||
(void) hts_isStringIDNA(s, size);
|
||||
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
|
||||
<meta> charset declaration. */
|
||||
#include "fuzz.h"
|
||||
#include "htscharset.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *html = fuzz_strdup(data, size);
|
||||
char *charset = hts_getCharsetFromMeta(html, size);
|
||||
|
||||
freet(charset);
|
||||
freet(html);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
|
||||
output buffer size, so the bounded-copy contract is exercised. */
|
||||
#include "fuzz.h"
|
||||
#include "httrack-library.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static const size_t bsizes[] = {1, 2, 16, 256, 8192};
|
||||
size_t bsize;
|
||||
char *s, *catbuff;
|
||||
|
||||
if (size == 0)
|
||||
return 0;
|
||||
bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
|
||||
data++, size--;
|
||||
s = fuzz_strdup(data, size);
|
||||
catbuff = malloct(bsize);
|
||||
|
||||
(void) unescape_http(catbuff, bsize, s);
|
||||
(void) unescape_http_unharm(catbuff, bsize, s, 0);
|
||||
(void) unescape_http_unharm(catbuff, bsize, s, 1);
|
||||
unescape_amp(s);
|
||||
|
||||
freet(catbuff);
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
|
||||
is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
|
||||
in place. */
|
||||
#include "fuzz.h"
|
||||
#include "htscore.h"
|
||||
#include "htslib.h"
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
char *s = fuzz_strdup(data, size);
|
||||
lien_adrfil *af = calloct(1, sizeof(*af));
|
||||
/* fil_simplifie rewrites in place and may grow an empty path to "./" */
|
||||
char *path = malloct(size + 3);
|
||||
|
||||
(void) ident_url_absolute(s, af);
|
||||
memcpy(path, s, size + 1);
|
||||
fil_simplifie(path);
|
||||
|
||||
freet(path);
|
||||
freet(af);
|
||||
freet(s);
|
||||
return 0;
|
||||
}
|
||||
51
fuzz/fuzz.h
51
fuzz/fuzz.h
@@ -1,51 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Shared helpers for the libFuzzer harnesses. */
|
||||
#ifndef FUZZ_H
|
||||
#define FUZZ_H
|
||||
|
||||
#define HTS_INTERNAL_BYTECODE
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "htsbase.h"
|
||||
|
||||
/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
|
||||
static char *fuzz_strdup(const uint8_t *data, size_t size) {
|
||||
char *s = malloct(size + 1);
|
||||
|
||||
memcpy(s, data, size);
|
||||
s[size] = '\0';
|
||||
return s;
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
|
||||
|
||||
#endif
|
||||
@@ -1,41 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Drive every built harness against its seed corpus.
|
||||
# run-fuzzers.sh <build-fuzz-dir> check deterministic replay (CI smoke)
|
||||
# run-fuzzers.sh <build-fuzz-dir> [seconds] timed mutation run (discovery)
|
||||
# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
|
||||
# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
|
||||
set -euo pipefail
|
||||
|
||||
srcdir=$(cd "$(dirname "$0")" && pwd)
|
||||
bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
|
||||
mode=${2:-20}
|
||||
|
||||
status=0
|
||||
for f in "$bld"/fuzz-*; do
|
||||
if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
|
||||
case "$f" in *.o | *.c | *.dSYM) continue ;; esac
|
||||
name=$(basename "$f")
|
||||
corpus="$srcdir/corpus/${name#fuzz-}"
|
||||
if [ "$mode" = "check" ]; then
|
||||
echo "=== $name (replay) ==="
|
||||
[ -d "$corpus" ] || continue
|
||||
if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
|
||||
echo "*** $name FAILED on its corpus" >&2
|
||||
status=1
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
work=$(mktemp -d)
|
||||
args=("$work")
|
||||
[ -d "$corpus" ] && args+=("$corpus")
|
||||
echo "=== $name (${mode}s) ==="
|
||||
if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
|
||||
-artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
|
||||
echo "*** $name FAILED; artifacts:" >&2
|
||||
ls -l "$work" >&2
|
||||
status=1
|
||||
else
|
||||
rm -rf "$work"
|
||||
fi
|
||||
done
|
||||
exit $status
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "07 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "27 June 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -51,7 +51,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%T, \-\-utf8\-conversion\fR ]
|
||||
[ \fB\-bN, \-\-cookies[=N]\fR ]
|
||||
[ \fB\-%K, \-\-cookies\-file\fR ]
|
||||
[ \fB\-%Y, \-\-why\fR ]
|
||||
[ \fB\-u, \-\-check\-type[=N]\fR ]
|
||||
[ \fB\-j, \-\-parse\-java[=N]\fR ]
|
||||
[ \fB\-sN, \-\-robots[=N]\fR ]
|
||||
@@ -219,8 +218,6 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
|
||||
accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
|
||||
.IP \-%K
|
||||
load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
|
||||
.IP \-%Y
|
||||
explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
|
||||
.IP \-u
|
||||
check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
|
||||
.IP \-j
|
||||
|
||||
@@ -114,8 +114,6 @@ const char *hts_optalias[][4] = {
|
||||
"strip [host/pattern=]key1,key2,... from URLs"},
|
||||
{"cookies-file", "-%K", "param1",
|
||||
"load extra cookies from a Netscape cookies.txt"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
"random pause of MIN[:MAX] seconds between files"},
|
||||
{"generate-errors", "-o", "single", ""},
|
||||
|
||||
@@ -910,19 +910,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
|
||||
/* Reader: can read bytes up to j */
|
||||
#define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
|
||||
|
||||
/* Writer: a malformed sequence abandons the encode (NULL) */
|
||||
#define WR(C) \
|
||||
do { \
|
||||
if ((C) != -1) { \
|
||||
/* copy character */ \
|
||||
assertf(segOutputSize < segSize); \
|
||||
segInt[segOutputSize++] = (C); \
|
||||
} else { \
|
||||
freet(segInt); \
|
||||
FREE_BUFFER(); \
|
||||
return NULL; \
|
||||
} \
|
||||
} while (0)
|
||||
/* Writer: upon error, return FFFD (replacement character) */
|
||||
#define WR(C) do { \
|
||||
if ((C) != -1) { \
|
||||
/* copy character */ \
|
||||
assertf(segOutputSize < segSize); \
|
||||
segInt[segOutputSize++] = (C); \
|
||||
} \
|
||||
/* In case of error, abort. */ \
|
||||
else { \
|
||||
FREE_BUFFER(); \
|
||||
return NULL; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* Read/Write Unicode character. */
|
||||
READ_UNICODE(RD, WR);
|
||||
@@ -1144,7 +1144,7 @@ int hts_isStringUTF8(const char *s, size_t size) {
|
||||
/* Reader: can read bytes up to j */
|
||||
#define RD ( i < size ? data[i++] : -1 )
|
||||
|
||||
/* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
|
||||
/* Writer: upon error, return FFFD (replacement character) */
|
||||
#define WR(C) if ((C) == -1) { return 0; }
|
||||
|
||||
/* Read Unicode character. */
|
||||
|
||||
@@ -846,52 +846,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
} // while
|
||||
|
||||
/* --why: print which filter rule decides for this URL, then stop */
|
||||
if (StringNotEmpty(opt->why_url)) {
|
||||
char BIGSTK url[HTS_URLMAXSIZE * 2];
|
||||
lien_adrfil af;
|
||||
|
||||
if (strstr(StringBuff(opt->why_url), ":/") == NULL)
|
||||
snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
|
||||
else
|
||||
strcpybuff(url, StringBuff(opt->why_url));
|
||||
if (ident_url_absolute(url, &af) < 0) {
|
||||
printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
|
||||
} else {
|
||||
char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
|
||||
int jok, jokDepth = 0;
|
||||
|
||||
/* the two forms the wizard tests */
|
||||
strcpybuff(l, jump_identification_const(af.adr));
|
||||
if (*af.fil != '/')
|
||||
strcatbuff(l, "/");
|
||||
strcatbuff(l, af.fil);
|
||||
if (!link_has_authority(af.adr))
|
||||
strcpybuff(lfull, "http://");
|
||||
else
|
||||
lfull[0] = '\0';
|
||||
strcatbuff(lfull, af.adr);
|
||||
if (*af.fil != '/')
|
||||
strcatbuff(lfull, "/");
|
||||
strcatbuff(lfull, af.fil);
|
||||
jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
|
||||
NULL, &jokDepth);
|
||||
if (jok > 0)
|
||||
printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
|
||||
filters[jokDepth]);
|
||||
else if (jok < 0)
|
||||
printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
|
||||
filters[jokDepth]);
|
||||
else
|
||||
printf("%s: no filter rule matches; the wizard decides "
|
||||
"(same-site links are followed by default)" LF,
|
||||
url);
|
||||
}
|
||||
freet(primary);
|
||||
XH_extuninit;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* load URL file list */
|
||||
/* OPTIMIZED for fast load */
|
||||
if (StringNotEmpty(opt->filelist)) {
|
||||
|
||||
@@ -1803,23 +1803,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->cookies_file, argv[na]);
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
printf(
|
||||
"Example: --why \"http://www.example.com/file.zip\"\n");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
na++;
|
||||
if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
|
||||
HTS_PANIC_PRINTF("why URL too long");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
StringCopy(opt->why_url, argv[na]);
|
||||
}
|
||||
break;
|
||||
case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option pause needs a blank space and a "
|
||||
|
||||
@@ -94,32 +94,6 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
|
||||
return verdict;
|
||||
}
|
||||
|
||||
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
|
||||
const char *nom2, LLint *size, int *size_flag,
|
||||
int *depth) {
|
||||
int depth1 = 0, depth2 = 0;
|
||||
int flag1 = 0, flag2 = 0;
|
||||
LLint sz1 = 0, sz2 = 0;
|
||||
int jok1, jok2, use1;
|
||||
|
||||
if (size) {
|
||||
sz1 = *size;
|
||||
sz2 = *size;
|
||||
}
|
||||
jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
|
||||
size_flag ? &flag1 : NULL, &depth1);
|
||||
jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
|
||||
size_flag ? &flag2 : NULL, &depth2);
|
||||
use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
|
||||
if (size)
|
||||
*size = use1 ? sz1 : sz2;
|
||||
if (size_flag)
|
||||
*size_flag = use1 ? flag1 : flag2;
|
||||
if (depth)
|
||||
*depth = use1 ? depth1 : depth2;
|
||||
return use1 ? jok1 : jok2;
|
||||
}
|
||||
|
||||
// supercomparateur joker (tm)
|
||||
// compare a et b (b=avec joker dedans), case insensitive [voir CI]
|
||||
// renvoi l'adresse de la première lettre de la chaine
|
||||
@@ -312,7 +286,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
if (!unique)
|
||||
max = (int) strlen(chaine);
|
||||
else /* *(a) only match a (not aaaaa) */
|
||||
max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
|
||||
max = 1;
|
||||
while(i < (int) max) {
|
||||
if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé
|
||||
if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
|
||||
|
||||
@@ -41,11 +41,6 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
|
||||
int *size_flag, int *depth);
|
||||
/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
|
||||
scheme); the match latest in the list wins, a "don't know" verdict defers.
|
||||
Returns the merged verdict; the out-params carry the winner's values. */
|
||||
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
|
||||
const char *nom2, LLint *size, int *size_flag, int *depth);
|
||||
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
|
||||
int *size_flag);
|
||||
const char *strjokerfind(const char *chaine, const char *joker);
|
||||
|
||||
@@ -543,7 +543,6 @@ void help(const char *app, int more) {
|
||||
infomsg("Spider options:");
|
||||
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
|
||||
infomsg(" %K load extra cookies from a Netscape cookies.txt");
|
||||
infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit");
|
||||
infomsg
|
||||
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
|
||||
infomsg
|
||||
|
||||
@@ -2497,10 +2497,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
|
||||
// effacer adrfil->adr et adrfil->fil
|
||||
adrfil->adr[0] = adrfil->fil[0] = '\0';
|
||||
|
||||
// Reject an over-long URL: a root-relative path is copied whole into fil[].
|
||||
if (strlen(url) >= HTS_URLMAXSIZE * 2)
|
||||
return -1;
|
||||
|
||||
#if HDEBUG
|
||||
printf("protocol: %s\n", url);
|
||||
#endif
|
||||
@@ -2576,7 +2572,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
|
||||
if (*p == '/' || *p == '\\') { /* adrfil->file:///.. */
|
||||
strcatbuff(adrfil->fil, p); // fichier local ; adrfil->adr="#"
|
||||
} else {
|
||||
if (*p == '\0' || p[1] != ':') { /* empty path: not a DOS drive letter */
|
||||
if (p[1] != ':') {
|
||||
strcatbuff(adrfil->fil, "//"); /* adrfil->file://server/foo */
|
||||
strcatbuff(adrfil->fil, p);
|
||||
} else {
|
||||
@@ -6006,7 +6002,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
|
||||
StringCopy(opt->strip_query, "");
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
opt->ftp_proxy = HTS_TRUE;
|
||||
@@ -6154,7 +6149,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
|
||||
StringFree(opt->mod_blacklist);
|
||||
StringFree(opt->strip_query);
|
||||
StringFree(opt->cookies_file);
|
||||
StringFree(opt->why_url);
|
||||
|
||||
StringFree(opt->path_html);
|
||||
StringFree(opt->path_html_utf8);
|
||||
|
||||
@@ -536,8 +536,6 @@ struct httrackp {
|
||||
(--cookies-file) */
|
||||
int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
|
||||
int pause_max_ms; /**< inter-file pause upper bound, ms */
|
||||
String why_url; /**< URL to diagnose (--why): print the deciding filter rule
|
||||
and exit without crawling */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
|
||||
@@ -579,16 +579,15 @@ static int st_filter(httrackp *opt, int argc, char **argv) {
|
||||
int matched;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "filter: needs a filter pattern and a string\n");
|
||||
return 1;
|
||||
}
|
||||
/* exact-size heap copies so a sanitizer traps an over-read; a missing
|
||||
subject means "" (not reachable as a CLI arg) */
|
||||
str = strdupt(argc >= 2 ? argv[1] : "");
|
||||
/* exact-size heap copies so a sanitizer traps any over-read of the pattern */
|
||||
str = strdupt(argv[1]);
|
||||
pat = strdupt(argv[0]);
|
||||
matched = strjoker(str, pat, NULL, NULL) != NULL;
|
||||
printf("%s does %s %s\n", str, matched ? "match" : "NOT match", argv[0]);
|
||||
printf("%s does %s %s\n", argv[1], matched ? "match" : "NOT match", argv[0]);
|
||||
freet(str);
|
||||
freet(pat);
|
||||
return 0;
|
||||
@@ -620,26 +619,6 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
|
||||
static int st_filterdual(httrackp *opt, int argc, char **argv) {
|
||||
int depth = -1, verdict;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 3) {
|
||||
fprintf(stderr,
|
||||
"filterdual: needs <string1> <string2> <filter> [filter...]\n");
|
||||
return 1;
|
||||
}
|
||||
verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
|
||||
NULL, &depth);
|
||||
printf("verdict=%s rule=%d\n",
|
||||
verdict > 0 ? "allowed"
|
||||
: verdict < 0 ? "forbidden"
|
||||
: "unknown",
|
||||
depth);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_simplify(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
@@ -1115,33 +1094,6 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Split a URL into (adr, fil), or print "error" if rejected. A second arg pads
|
||||
the URL with that many 'a's to reach lengths a CLI arg can't. */
|
||||
static int st_identurl(httrackp *opt, int argc, char **argv) {
|
||||
lien_adrfil af;
|
||||
char *url;
|
||||
size_t len, pad = 0;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "identurl: needs a URL\n");
|
||||
return 1;
|
||||
}
|
||||
if (argc >= 2)
|
||||
pad = (size_t) atoi(argv[1]);
|
||||
len = strlen(argv[0]);
|
||||
url = malloct(len + pad + 1);
|
||||
memcpy(url, argv[0], len);
|
||||
memset(url + len, 'a', pad);
|
||||
url[len + pad] = '\0';
|
||||
if (ident_url_absolute(url, &af) >= 0)
|
||||
printf("adr=%s fil=%s\n", af.adr, af.fil);
|
||||
else
|
||||
printf("error\n");
|
||||
freet(url);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
|
||||
no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
|
||||
registering an already-crawled link (dedup/collision paths). */
|
||||
@@ -2154,8 +2106,6 @@ static const struct selftest_entry {
|
||||
{"filtersize", "<size> <string> <filter>...",
|
||||
"size-aware filter verdict (negative size = unknown/scan time)",
|
||||
st_filtersize},
|
||||
{"filterdual", "<string1> <string2> <filter>...",
|
||||
"merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
|
||||
{"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
|
||||
{"stripquery", "", "--strip-query pattern/key stripping self-test",
|
||||
st_stripquery},
|
||||
@@ -2182,7 +2132,6 @@ static const struct selftest_entry {
|
||||
st_relative},
|
||||
{"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
|
||||
st_resolve},
|
||||
{"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
|
||||
{"header", "<raw-header-line> ...", "response header-line parsing",
|
||||
st_header},
|
||||
{"savename", "<fil> <content-type> [key=value ...]",
|
||||
|
||||
@@ -524,11 +524,28 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
// filters, 0=sait pas 1=ok -1=interdit
|
||||
{
|
||||
int jokDepth = 0;
|
||||
int jokDepth1 = 0, jokDepth2 = 0;
|
||||
int jok1 = 0, jok2 = 0;
|
||||
|
||||
jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
|
||||
NULL, NULL, &jokDepth);
|
||||
mdepth = _FILTERS[jokDepth];
|
||||
jok1 =
|
||||
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
|
||||
&jokDepth1);
|
||||
jok2 =
|
||||
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
|
||||
&jokDepth2);
|
||||
if (jok2 == 0) { // #2 doesn't know
|
||||
jok = jok1; // then, use #1
|
||||
mdepth = _FILTERS[jokDepth1];
|
||||
} else if (jok1 == 0) { // #1 doesn't know
|
||||
jok = jok2; // then, use #2
|
||||
mdepth = _FILTERS[jokDepth2];
|
||||
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
|
||||
jok = jok1;
|
||||
mdepth = _FILTERS[jokDepth1];
|
||||
} else { // #2 matching rule is "after" #1, then it is prioritary
|
||||
jok = jok2;
|
||||
mdepth = _FILTERS[jokDepth2];
|
||||
}
|
||||
}
|
||||
|
||||
if (jok == 1) { // autorisé
|
||||
@@ -948,10 +965,34 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
|
||||
|
||||
// filters, 0=sait pas 1=ok -1=interdit
|
||||
{
|
||||
sz = size;
|
||||
jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
|
||||
*opt->filters.filptr, lfull, l, &sz, &size_flag,
|
||||
NULL);
|
||||
int jokDepth1 = 0, jokDepth2 = 0;
|
||||
int jok1 = 0, jok2 = 0;
|
||||
LLint sz1 = size, sz2 = size;
|
||||
int size_flag1 = 0, size_flag2 = 0;
|
||||
|
||||
jok1 =
|
||||
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
|
||||
lfull, &sz1, &size_flag1, &jokDepth1);
|
||||
jok2 =
|
||||
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
|
||||
l, &sz2, &size_flag2, &jokDepth2);
|
||||
if (jok2 == 0) { // #2 doesn't know
|
||||
jok = jok1; // then, use #1
|
||||
sz = sz1;
|
||||
size_flag = size_flag1;
|
||||
} else if (jok1 == 0) { // #1 doesn't know
|
||||
jok = jok2; // then, use #2
|
||||
sz = sz2;
|
||||
size_flag = size_flag2;
|
||||
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
|
||||
jok = jok1;
|
||||
sz = sz1;
|
||||
size_flag = size_flag1;
|
||||
} else { // #2 matching rule is "after" #1, then it is prioritary
|
||||
jok = jok2;
|
||||
sz = sz2;
|
||||
size_flag = size_flag2;
|
||||
}
|
||||
}
|
||||
|
||||
// log
|
||||
|
||||
@@ -12,11 +12,6 @@ match() {
|
||||
nomatch() {
|
||||
test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does NOT match $1" || exit 1
|
||||
}
|
||||
# single-arg call means an empty subject (unreachable as a CLI arg); '*(' used
|
||||
# to force max=1 and over-read past it.
|
||||
nomatch_empty() {
|
||||
test "$(httrack -O /dev/null -#test=filter "$1")" == " does NOT match $1" || exit 1
|
||||
}
|
||||
|
||||
# bare star matches everything
|
||||
match '*' 'anything/at/all'
|
||||
@@ -127,7 +122,3 @@ nomatch '*[file]end' 'foo?xend'
|
||||
nomatch '*[name]X' 'abc?X'
|
||||
match '*[file]' 'foo?x=1' # trailing query: tolerated, as for *.aspx
|
||||
match '*.aspx' 'page.aspx?y=2'
|
||||
|
||||
# empty subject against a unique-match allow-all pattern (heap over-read regress)
|
||||
nomatch_empty '*(('
|
||||
nomatch_empty '**(('
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
|
||||
# the match latest in the filter list wins, an unknown verdict defers.
|
||||
|
||||
fdual() {
|
||||
local want="$1"
|
||||
shift
|
||||
test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
|
||||
}
|
||||
|
||||
fdual 'verdict=unknown rule=0' zzz yyy '+a*' # neither form matches
|
||||
fdual 'verdict=forbidden rule=0' aaa zzz '-a*' # only form 1 matches
|
||||
fdual 'verdict=forbidden rule=0' zzz aaa '-a*' # only form 2 matches
|
||||
fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
|
||||
fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
|
||||
fdual 'verdict=allowed rule=0' a aa '+a*' # tie: form 1 wins
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ident_url_absolute splits a raw URL into (adr, fil), the first parser to touch
|
||||
# it. -#test=identurl <url> [pad] prints "adr=.. fil=.." or "error"; pad appends
|
||||
# N 'a's, reaching lengths a CLI arg can't.
|
||||
|
||||
split() {
|
||||
test "$(httrack -O /dev/null -#test=identurl "$1")" == "$2" || exit 1
|
||||
}
|
||||
|
||||
split 'http://www.example.com/a/b/../c.html' 'adr=www.example.com fil=/a/c.html'
|
||||
split 'ftp://ftp.example.com/pub/file.txt' 'adr=ftp://ftp.example.com fil=/pub/file.txt'
|
||||
split 'file:///etc/hostname' 'adr=file:// fil=/etc/hostname'
|
||||
|
||||
# empty-path file:// used to read one byte past the URL (p[1] with *p=0)
|
||||
split 'file://' 'adr=file:// fil=//'
|
||||
|
||||
# a root-relative path is copied whole into fil[HTS_URLMAXSIZE*2], so a URL at
|
||||
# the buffer size is the tightest overflow; it must be rejected, not abort.
|
||||
split_pad() {
|
||||
test "$(httrack -O /dev/null -#test=identurl "$1" "$2")" == "$3" || exit 1
|
||||
}
|
||||
split_pad '/' 2047 'error' # 1 + 2047 = 2048 = HTS_URLMAXSIZE*2
|
||||
split_pad 'http://h/' 3000 'error'
|
||||
@@ -8,8 +8,8 @@ set -euo pipefail
|
||||
|
||||
enc() { test "$(httrack -O /dev/null -#test=idna-encode "$1")" == "$2" || exit 1; }
|
||||
dec() { test "$(httrack -O /dev/null -#test=idna-decode "$1")" == "$2" || exit 1; }
|
||||
# a malformed encode input must be rejected (empty output), not encoded or leaked.
|
||||
enc_bad() { test -z "$(httrack -O /dev/null -#test=idna-encode "$1" 2>/dev/null)" || exit 1; }
|
||||
# crash probe: malformed ACE input must exit cleanly, not abort.
|
||||
runs() { httrack -O /dev/null -#test=idna-decode "$1" >/dev/null 2>&1 || exit 1; }
|
||||
|
||||
# encode
|
||||
enc 'www.café.com' 'www.xn--caf-dma.com'
|
||||
@@ -33,9 +33,6 @@ enc 'xn--already-encoded.com' 'xn--already-encoded.com'
|
||||
# an empty punycode payload decodes back to the bare xn-- label
|
||||
dec 'xn--' 'xn--'
|
||||
|
||||
# malformed ACE payloads decode to a stable (garbage) string, not a crash
|
||||
dec 'xn--!!!' '㚯'
|
||||
dec 'xn--already-encoded.com' 'ǮalǭǮreaǭǫdy.com'
|
||||
|
||||
# second label has a truncated UTF-8 sequence: used to leak the segment buffer.
|
||||
enc_bad "$(printf 'b\xc3\xbcchev.\xe4\xbe\x8b\xe5\xad\x62\xc3\xbccheple')"
|
||||
# malformed ACE payloads (invalid base-36, garbage) must not crash
|
||||
runs 'xn--!!!'
|
||||
runs 'xn--already-encoded.com'
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --why: report which +/- filter rule decides for a URL, without crawling.
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmpdir"' EXIT
|
||||
|
||||
why() {
|
||||
local want="$1" out
|
||||
shift
|
||||
out="$(httrack -O "$tmpdir/p" -q "$@" |
|
||||
grep -E 'accepted|rejected|no filter|unable')"
|
||||
test "$out" == "$want" || {
|
||||
echo "want: $want"
|
||||
echo "got : $out"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
base=http://www.example.com
|
||||
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
|
||||
--why "$base/x.zip" "$base/" '+*.png' '-*.zip'
|
||||
why "$base/a.png: accepted by rule #1 (+*.png)" \
|
||||
--why "$base/a.png" "$base/" '+*.png' '-*.zip'
|
||||
why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
|
||||
--why "$base/i.html" "$base/" '+*.png' '-*.zip'
|
||||
# scheme-less input gets the same http:// default as primary URLs
|
||||
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
|
||||
--why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
|
||||
# the rule latest in the list decides
|
||||
why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
|
||||
--why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'
|
||||
@@ -35,13 +35,11 @@ TESTS = \
|
||||
01_engine-entities.test \
|
||||
01_engine-filelist.test \
|
||||
01_engine-filter.test \
|
||||
01_engine-filterdual.test \
|
||||
01_engine-ftp-line.test \
|
||||
01_engine-ftp-userpass.test \
|
||||
01_engine-hashtable.test \
|
||||
01_engine-header.test \
|
||||
01_engine-idna.test \
|
||||
01_engine-identurl.test \
|
||||
01_engine-escape-room.test \
|
||||
01_engine-inplace-escape.test \
|
||||
01_engine-java.test \
|
||||
@@ -106,7 +104,6 @@ TESTS = \
|
||||
36_local-bigcrawl.test \
|
||||
37_local-cache-outage.test \
|
||||
38_local-update-304.test \
|
||||
39_local-delayed-cancel.test \
|
||||
40_local-why.test
|
||||
39_local-delayed-cancel.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
Reference in New Issue
Block a user