mirror of
https://github.com/xroche/httrack.git
synced 2026-07-08 01:46:21 +03:00
Compare commits
1 Commits
master
...
p3-3-codeq
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d290fb5fc |
10
configure.ac
10
configure.ac
@@ -102,8 +102,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
|
||||
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
|
||||
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
|
||||
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
|
||||
@@ -111,13 +110,6 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
|
||||
|
||||
# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
|
||||
# sanitizer builds, whose interceptors want the unfortified calls.
|
||||
case "$CFLAGS" in
|
||||
*-fsanitize=*) ;;
|
||||
*) AX_ADD_FORTIFY_SOURCE ;;
|
||||
esac
|
||||
|
||||
# Force libc back into DT_NEEDED for libraries that reach it only through
|
||||
# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
|
||||
# linker; Apple ld rejects these flags and links libSystem unconditionally.
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
# ===========================================================================
|
||||
# https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
|
||||
# ===========================================================================
|
||||
#
|
||||
# SYNOPSIS
|
||||
#
|
||||
# AX_ADD_FORTIFY_SOURCE
|
||||
#
|
||||
# DESCRIPTION
|
||||
#
|
||||
# Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
|
||||
# redefinition warnings, other cpp warnings or linker. Some distributions
|
||||
# (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
|
||||
# their compilers, leading to unnecessary warnings in the form of
|
||||
#
|
||||
# <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
|
||||
# <built-in>: note: this is the location of the previous definition
|
||||
#
|
||||
# which is a problem if -Werror is enabled. This macro checks whether
|
||||
# _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
|
||||
# to CPPFLAGS.
|
||||
#
|
||||
# Newer mingw-w64 msys2 package comes with a bug in
|
||||
# headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
|
||||
# and would need -lssp or -fstack-protector. See
|
||||
# https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
|
||||
# link it.
|
||||
#
|
||||
# LICENSE
|
||||
#
|
||||
# Copyright (c) 2017 David Seifert <soap@gentoo.org>
|
||||
# Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
|
||||
#
|
||||
# Copying and distribution of this file, with or without modification, are
|
||||
# permitted in any medium without royalty provided the copyright notice
|
||||
# and this notice are preserved. This file is offered as-is, without any
|
||||
# warranty.
|
||||
|
||||
#serial 10
|
||||
|
||||
AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
|
||||
ac_save_cflags=$CFLAGS
|
||||
ac_cwerror_flag=yes
|
||||
AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
|
||||
ax_add_fortify_3_failed=
|
||||
AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
|
||||
AC_LINK_IFELSE([
|
||||
AC_LANG_PROGRAM([],
|
||||
[[
|
||||
#ifndef _FORTIFY_SOURCE
|
||||
return 0;
|
||||
#else
|
||||
_FORTIFY_SOURCE_already_defined;
|
||||
#endif
|
||||
]]
|
||||
)],
|
||||
AC_LINK_IFELSE([
|
||||
AC_LANG_SOURCE([[
|
||||
#define _FORTIFY_SOURCE 3
|
||||
#include <string.h>
|
||||
int main(void) {
|
||||
char *s = " ";
|
||||
strcpy(s, "x");
|
||||
return strlen(s)-1;
|
||||
}
|
||||
]]
|
||||
)],
|
||||
[
|
||||
AC_MSG_RESULT([yes])
|
||||
CFLAGS=$ac_save_cflags
|
||||
CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
|
||||
], [
|
||||
AC_MSG_RESULT([no])
|
||||
ax_add_fortify_3_failed=1
|
||||
],
|
||||
),
|
||||
[
|
||||
AC_MSG_RESULT([no])
|
||||
ax_add_fortify_3_failed=1
|
||||
])
|
||||
if test -n "$ax_add_fortify_3_failed"
|
||||
then
|
||||
AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
|
||||
AC_LINK_IFELSE([
|
||||
AC_LANG_PROGRAM([],
|
||||
[[
|
||||
#ifndef _FORTIFY_SOURCE
|
||||
return 0;
|
||||
#else
|
||||
_FORTIFY_SOURCE_already_defined;
|
||||
#endif
|
||||
]]
|
||||
)],
|
||||
AC_LINK_IFELSE([
|
||||
AC_LANG_SOURCE([[
|
||||
#define _FORTIFY_SOURCE 2
|
||||
#include <string.h>
|
||||
int main(void) {
|
||||
char *s = " ";
|
||||
strcpy(s, "x");
|
||||
return strlen(s)-1;
|
||||
}
|
||||
]]
|
||||
)],
|
||||
[
|
||||
AC_MSG_RESULT([yes])
|
||||
CFLAGS=$ac_save_cflags
|
||||
CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
|
||||
], [
|
||||
AC_MSG_RESULT([no])
|
||||
CFLAGS=$ac_save_cflags
|
||||
],
|
||||
),
|
||||
[
|
||||
AC_MSG_RESULT([no])
|
||||
CFLAGS=$ac_save_cflags
|
||||
])
|
||||
fi
|
||||
])
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "07 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "27 June 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -51,7 +51,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%T, \-\-utf8\-conversion\fR ]
|
||||
[ \fB\-bN, \-\-cookies[=N]\fR ]
|
||||
[ \fB\-%K, \-\-cookies\-file\fR ]
|
||||
[ \fB\-%Y, \-\-why\fR ]
|
||||
[ \fB\-u, \-\-check\-type[=N]\fR ]
|
||||
[ \fB\-j, \-\-parse\-java[=N]\fR ]
|
||||
[ \fB\-sN, \-\-robots[=N]\fR ]
|
||||
@@ -219,8 +218,6 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
|
||||
accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
|
||||
.IP \-%K
|
||||
load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
|
||||
.IP \-%Y
|
||||
explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
|
||||
.IP \-u
|
||||
check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
|
||||
.IP \-j
|
||||
|
||||
@@ -114,8 +114,6 @@ const char *hts_optalias[][4] = {
|
||||
"strip [host/pattern=]key1,key2,... from URLs"},
|
||||
{"cookies-file", "-%K", "param1",
|
||||
"load extra cookies from a Netscape cookies.txt"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
"random pause of MIN[:MAX] seconds between files"},
|
||||
{"generate-errors", "-o", "single", ""},
|
||||
|
||||
@@ -846,52 +846,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
} // while
|
||||
|
||||
/* --why: print which filter rule decides for this URL, then stop */
|
||||
if (StringNotEmpty(opt->why_url)) {
|
||||
char BIGSTK url[HTS_URLMAXSIZE * 2];
|
||||
lien_adrfil af;
|
||||
|
||||
if (strstr(StringBuff(opt->why_url), ":/") == NULL)
|
||||
snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
|
||||
else
|
||||
strcpybuff(url, StringBuff(opt->why_url));
|
||||
if (ident_url_absolute(url, &af) < 0) {
|
||||
printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
|
||||
} else {
|
||||
char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
|
||||
int jok, jokDepth = 0;
|
||||
|
||||
/* the two forms the wizard tests */
|
||||
strcpybuff(l, jump_identification_const(af.adr));
|
||||
if (*af.fil != '/')
|
||||
strcatbuff(l, "/");
|
||||
strcatbuff(l, af.fil);
|
||||
if (!link_has_authority(af.adr))
|
||||
strcpybuff(lfull, "http://");
|
||||
else
|
||||
lfull[0] = '\0';
|
||||
strcatbuff(lfull, af.adr);
|
||||
if (*af.fil != '/')
|
||||
strcatbuff(lfull, "/");
|
||||
strcatbuff(lfull, af.fil);
|
||||
jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
|
||||
NULL, &jokDepth);
|
||||
if (jok > 0)
|
||||
printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
|
||||
filters[jokDepth]);
|
||||
else if (jok < 0)
|
||||
printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
|
||||
filters[jokDepth]);
|
||||
else
|
||||
printf("%s: no filter rule matches; the wizard decides "
|
||||
"(same-site links are followed by default)" LF,
|
||||
url);
|
||||
}
|
||||
freet(primary);
|
||||
XH_extuninit;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* load URL file list */
|
||||
/* OPTIMIZED for fast load */
|
||||
if (StringNotEmpty(opt->filelist)) {
|
||||
|
||||
@@ -1803,23 +1803,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->cookies_file, argv[na]);
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
printf(
|
||||
"Example: --why \"http://www.example.com/file.zip\"\n");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
na++;
|
||||
if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
|
||||
HTS_PANIC_PRINTF("why URL too long");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
StringCopy(opt->why_url, argv[na]);
|
||||
}
|
||||
break;
|
||||
case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option pause needs a blank space and a "
|
||||
|
||||
@@ -94,32 +94,6 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
|
||||
return verdict;
|
||||
}
|
||||
|
||||
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
|
||||
const char *nom2, LLint *size, int *size_flag,
|
||||
int *depth) {
|
||||
int depth1 = 0, depth2 = 0;
|
||||
int flag1 = 0, flag2 = 0;
|
||||
LLint sz1 = 0, sz2 = 0;
|
||||
int jok1, jok2, use1;
|
||||
|
||||
if (size) {
|
||||
sz1 = *size;
|
||||
sz2 = *size;
|
||||
}
|
||||
jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
|
||||
size_flag ? &flag1 : NULL, &depth1);
|
||||
jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
|
||||
size_flag ? &flag2 : NULL, &depth2);
|
||||
use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
|
||||
if (size)
|
||||
*size = use1 ? sz1 : sz2;
|
||||
if (size_flag)
|
||||
*size_flag = use1 ? flag1 : flag2;
|
||||
if (depth)
|
||||
*depth = use1 ? depth1 : depth2;
|
||||
return use1 ? jok1 : jok2;
|
||||
}
|
||||
|
||||
// supercomparateur joker (tm)
|
||||
// compare a et b (b=avec joker dedans), case insensitive [voir CI]
|
||||
// renvoi l'adresse de la première lettre de la chaine
|
||||
|
||||
@@ -41,11 +41,6 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
|
||||
int *size_flag, int *depth);
|
||||
/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
|
||||
scheme); the match latest in the list wins, a "don't know" verdict defers.
|
||||
Returns the merged verdict; the out-params carry the winner's values. */
|
||||
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
|
||||
const char *nom2, LLint *size, int *size_flag, int *depth);
|
||||
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
|
||||
int *size_flag);
|
||||
const char *strjokerfind(const char *chaine, const char *joker);
|
||||
|
||||
@@ -543,7 +543,6 @@ void help(const char *app, int more) {
|
||||
infomsg("Spider options:");
|
||||
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
|
||||
infomsg(" %K load extra cookies from a Netscape cookies.txt");
|
||||
infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit");
|
||||
infomsg
|
||||
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
|
||||
infomsg
|
||||
|
||||
@@ -6006,7 +6006,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
|
||||
StringCopy(opt->strip_query, "");
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
opt->ftp_proxy = HTS_TRUE;
|
||||
@@ -6154,7 +6153,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
|
||||
StringFree(opt->mod_blacklist);
|
||||
StringFree(opt->strip_query);
|
||||
StringFree(opt->cookies_file);
|
||||
StringFree(opt->why_url);
|
||||
|
||||
StringFree(opt->path_html);
|
||||
StringFree(opt->path_html_utf8);
|
||||
|
||||
@@ -536,8 +536,6 @@ struct httrackp {
|
||||
(--cookies-file) */
|
||||
int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
|
||||
int pause_max_ms; /**< inter-file pause upper bound, ms */
|
||||
String why_url; /**< URL to diagnose (--why): print the deciding filter rule
|
||||
and exit without crawling */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
|
||||
@@ -620,26 +620,6 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
|
||||
static int st_filterdual(httrackp *opt, int argc, char **argv) {
|
||||
int depth = -1, verdict;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 3) {
|
||||
fprintf(stderr,
|
||||
"filterdual: needs <string1> <string2> <filter> [filter...]\n");
|
||||
return 1;
|
||||
}
|
||||
verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
|
||||
NULL, &depth);
|
||||
printf("verdict=%s rule=%d\n",
|
||||
verdict > 0 ? "allowed"
|
||||
: verdict < 0 ? "forbidden"
|
||||
: "unknown",
|
||||
depth);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_simplify(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
@@ -2154,8 +2134,6 @@ static const struct selftest_entry {
|
||||
{"filtersize", "<size> <string> <filter>...",
|
||||
"size-aware filter verdict (negative size = unknown/scan time)",
|
||||
st_filtersize},
|
||||
{"filterdual", "<string1> <string2> <filter>...",
|
||||
"merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
|
||||
{"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
|
||||
{"stripquery", "", "--strip-query pattern/key stripping self-test",
|
||||
st_stripquery},
|
||||
|
||||
@@ -524,11 +524,28 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
// filters, 0=sait pas 1=ok -1=interdit
|
||||
{
|
||||
int jokDepth = 0;
|
||||
int jokDepth1 = 0, jokDepth2 = 0;
|
||||
int jok1 = 0, jok2 = 0;
|
||||
|
||||
jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
|
||||
NULL, NULL, &jokDepth);
|
||||
mdepth = _FILTERS[jokDepth];
|
||||
jok1 =
|
||||
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
|
||||
&jokDepth1);
|
||||
jok2 =
|
||||
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
|
||||
&jokDepth2);
|
||||
if (jok2 == 0) { // #2 doesn't know
|
||||
jok = jok1; // then, use #1
|
||||
mdepth = _FILTERS[jokDepth1];
|
||||
} else if (jok1 == 0) { // #1 doesn't know
|
||||
jok = jok2; // then, use #2
|
||||
mdepth = _FILTERS[jokDepth2];
|
||||
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
|
||||
jok = jok1;
|
||||
mdepth = _FILTERS[jokDepth1];
|
||||
} else { // #2 matching rule is "after" #1, then it is prioritary
|
||||
jok = jok2;
|
||||
mdepth = _FILTERS[jokDepth2];
|
||||
}
|
||||
}
|
||||
|
||||
if (jok == 1) { // autorisé
|
||||
@@ -948,10 +965,34 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
|
||||
|
||||
// filters, 0=sait pas 1=ok -1=interdit
|
||||
{
|
||||
sz = size;
|
||||
jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
|
||||
*opt->filters.filptr, lfull, l, &sz, &size_flag,
|
||||
NULL);
|
||||
int jokDepth1 = 0, jokDepth2 = 0;
|
||||
int jok1 = 0, jok2 = 0;
|
||||
LLint sz1 = size, sz2 = size;
|
||||
int size_flag1 = 0, size_flag2 = 0;
|
||||
|
||||
jok1 =
|
||||
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
|
||||
lfull, &sz1, &size_flag1, &jokDepth1);
|
||||
jok2 =
|
||||
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
|
||||
l, &sz2, &size_flag2, &jokDepth2);
|
||||
if (jok2 == 0) { // #2 doesn't know
|
||||
jok = jok1; // then, use #1
|
||||
sz = sz1;
|
||||
size_flag = size_flag1;
|
||||
} else if (jok1 == 0) { // #1 doesn't know
|
||||
jok = jok2; // then, use #2
|
||||
sz = sz2;
|
||||
size_flag = size_flag2;
|
||||
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
|
||||
jok = jok1;
|
||||
sz = sz1;
|
||||
size_flag = size_flag1;
|
||||
} else { // #2 matching rule is "after" #1, then it is prioritary
|
||||
jok = jok2;
|
||||
sz = sz2;
|
||||
size_flag = size_flag2;
|
||||
}
|
||||
}
|
||||
|
||||
// log
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
|
||||
# the match latest in the filter list wins, an unknown verdict defers.
|
||||
|
||||
fdual() {
|
||||
local want="$1"
|
||||
shift
|
||||
test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
|
||||
}
|
||||
|
||||
fdual 'verdict=unknown rule=0' zzz yyy '+a*' # neither form matches
|
||||
fdual 'verdict=forbidden rule=0' aaa zzz '-a*' # only form 1 matches
|
||||
fdual 'verdict=forbidden rule=0' zzz aaa '-a*' # only form 2 matches
|
||||
fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
|
||||
fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
|
||||
fdual 'verdict=allowed rule=0' a aa '+a*' # tie: form 1 wins
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --why: report which +/- filter rule decides for a URL, without crawling.
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmpdir"' EXIT
|
||||
|
||||
why() {
|
||||
local want="$1" out
|
||||
shift
|
||||
out="$(httrack -O "$tmpdir/p" -q "$@" |
|
||||
grep -E 'accepted|rejected|no filter|unable')"
|
||||
test "$out" == "$want" || {
|
||||
echo "want: $want"
|
||||
echo "got : $out"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
base=http://www.example.com
|
||||
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
|
||||
--why "$base/x.zip" "$base/" '+*.png' '-*.zip'
|
||||
why "$base/a.png: accepted by rule #1 (+*.png)" \
|
||||
--why "$base/a.png" "$base/" '+*.png' '-*.zip'
|
||||
why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
|
||||
--why "$base/i.html" "$base/" '+*.png' '-*.zip'
|
||||
# scheme-less input gets the same http:// default as primary URLs
|
||||
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
|
||||
--why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
|
||||
# the rule latest in the list decides
|
||||
why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
|
||||
--why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'
|
||||
@@ -35,7 +35,6 @@ TESTS = \
|
||||
01_engine-entities.test \
|
||||
01_engine-filelist.test \
|
||||
01_engine-filter.test \
|
||||
01_engine-filterdual.test \
|
||||
01_engine-ftp-line.test \
|
||||
01_engine-ftp-userpass.test \
|
||||
01_engine-hashtable.test \
|
||||
@@ -106,7 +105,6 @@ TESTS = \
|
||||
36_local-bigcrawl.test \
|
||||
37_local-cache-outage.test \
|
||||
38_local-update-304.test \
|
||||
39_local-delayed-cancel.test \
|
||||
40_local-why.test
|
||||
39_local-delayed-cancel.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
Reference in New Issue
Block a user