Compare commits

..

1 Commits

Author SHA1 Message Date
Xavier Roche
1d290fb5fc Add a CodeQL C analysis workflow
Scan on push to master, PRs, and a weekly cron, with the security-extended
query suite; findings land in the code-scanning dashboard. Manual build mode
so CodeQL traces the real autotools build.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-07 19:27:41 +02:00
16 changed files with 52 additions and 321 deletions

View File

@@ -102,8 +102,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -111,13 +110,6 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
# sanitizer builds, whose interceptors want the unfortified calls.
case "$CFLAGS" in
*-fsanitize=*) ;;
*) AX_ADD_FORTIFY_SOURCE ;;
esac
# Force libc back into DT_NEEDED for libraries that reach it only through
# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
# linker; Apple ld rejects these flags and links libSystem unconditionally.

View File

@@ -1,119 +0,0 @@
# ===========================================================================
# https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_ADD_FORTIFY_SOURCE
#
# DESCRIPTION
#
# Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
# redefinition warnings, other cpp warnings or linker. Some distributions
# (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
# their compilers, leading to unnecessary warnings in the form of
#
# <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
# <built-in>: note: this is the location of the previous definition
#
# which is a problem if -Werror is enabled. This macro checks whether
# _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
# to CPPFLAGS.
#
# Newer mingw-w64 msys2 package comes with a bug in
# headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
# and would need -lssp or -fstack-protector. See
# https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
# link it.
#
# LICENSE
#
# Copyright (c) 2017 David Seifert <soap@gentoo.org>
# Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved. This file is offered as-is, without any
# warranty.
#serial 10
AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
ac_save_cflags=$CFLAGS
ac_cwerror_flag=yes
AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
ax_add_fortify_3_failed=
AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
AC_LINK_IFELSE([
AC_LANG_PROGRAM([],
[[
#ifndef _FORTIFY_SOURCE
return 0;
#else
_FORTIFY_SOURCE_already_defined;
#endif
]]
)],
AC_LINK_IFELSE([
AC_LANG_SOURCE([[
#define _FORTIFY_SOURCE 3
#include <string.h>
int main(void) {
char *s = " ";
strcpy(s, "x");
return strlen(s)-1;
}
]]
)],
[
AC_MSG_RESULT([yes])
CFLAGS=$ac_save_cflags
CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
], [
AC_MSG_RESULT([no])
ax_add_fortify_3_failed=1
],
),
[
AC_MSG_RESULT([no])
ax_add_fortify_3_failed=1
])
if test -n "$ax_add_fortify_3_failed"
then
AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
AC_LINK_IFELSE([
AC_LANG_PROGRAM([],
[[
#ifndef _FORTIFY_SOURCE
return 0;
#else
_FORTIFY_SOURCE_already_defined;
#endif
]]
)],
AC_LINK_IFELSE([
AC_LANG_SOURCE([[
#define _FORTIFY_SOURCE 2
#include <string.h>
int main(void) {
char *s = " ";
strcpy(s, "x");
return strlen(s)-1;
}
]]
)],
[
AC_MSG_RESULT([yes])
CFLAGS=$ac_save_cflags
CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
], [
AC_MSG_RESULT([no])
CFLAGS=$ac_save_cflags
],
),
[
AC_MSG_RESULT([no])
CFLAGS=$ac_save_cflags
])
fi
])

View File

@@ -3,7 +3,7 @@
.\"
.\" This file is generated by man/makeman.sh; do not edit by hand.
.\" SPDX-License-Identifier: GPL-3.0-or-later
.TH httrack 1 "07 July 2026" "httrack website copier"
.TH httrack 1 "27 June 2026" "httrack website copier"
.SH NAME
httrack \- offline browser : copy websites to a local directory
.SH SYNOPSIS
@@ -51,7 +51,6 @@ httrack \- offline browser : copy websites to a local directory
[ \fB\-%T, \-\-utf8\-conversion\fR ]
[ \fB\-bN, \-\-cookies[=N]\fR ]
[ \fB\-%K, \-\-cookies\-file\fR ]
[ \fB\-%Y, \-\-why\fR ]
[ \fB\-u, \-\-check\-type[=N]\fR ]
[ \fB\-j, \-\-parse\-java[=N]\fR ]
[ \fB\-sN, \-\-robots[=N]\fR ]
@@ -219,8 +218,6 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
.IP \-%K
load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
.IP \-%Y
explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
.IP \-u
check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
.IP \-j

View File

@@ -114,8 +114,6 @@ const char *hts_optalias[][4] = {
"strip [host/pattern=]key1,key2,... from URLs"},
{"cookies-file", "-%K", "param1",
"load extra cookies from a Netscape cookies.txt"},
{"why", "-%Y", "param1",
"explain which filter rule accepts or rejects a URL, then exit"},
{"pause", "-%G", "param1",
"random pause of MIN[:MAX] seconds between files"},
{"generate-errors", "-o", "single", ""},

View File

@@ -846,52 +846,6 @@ int httpmirror(char *url1, httrackp * opt) {
}
} // while
/* --why: print which filter rule decides for this URL, then stop */
if (StringNotEmpty(opt->why_url)) {
char BIGSTK url[HTS_URLMAXSIZE * 2];
lien_adrfil af;
if (strstr(StringBuff(opt->why_url), ":/") == NULL)
snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
else
strcpybuff(url, StringBuff(opt->why_url));
if (ident_url_absolute(url, &af) < 0) {
printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
} else {
char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
int jok, jokDepth = 0;
/* the two forms the wizard tests */
strcpybuff(l, jump_identification_const(af.adr));
if (*af.fil != '/')
strcatbuff(l, "/");
strcatbuff(l, af.fil);
if (!link_has_authority(af.adr))
strcpybuff(lfull, "http://");
else
lfull[0] = '\0';
strcatbuff(lfull, af.adr);
if (*af.fil != '/')
strcatbuff(lfull, "/");
strcatbuff(lfull, af.fil);
jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
NULL, &jokDepth);
if (jok > 0)
printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
filters[jokDepth]);
else if (jok < 0)
printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
filters[jokDepth]);
else
printf("%s: no filter rule matches; the wizard decides "
"(same-site links are followed by default)" LF,
url);
}
freet(primary);
XH_extuninit;
return 1;
}
/* load URL file list */
/* OPTIMIZED for fast load */
if (StringNotEmpty(opt->filelist)) {

View File

@@ -1803,23 +1803,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
StringCopy(opt->cookies_file, argv[na]);
}
break;
case 'Y': // why: explain the filter verdict for a URL, no crawl
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
printf(
"Example: --why \"http://www.example.com/file.zip\"\n");
htsmain_free();
return -1;
} else {
na++;
if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
HTS_PANIC_PRINTF("why URL too long");
htsmain_free();
return -1;
}
StringCopy(opt->why_url, argv[na]);
}
break;
case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
HTS_PANIC_PRINTF("Option pause needs a blank space and a "

View File

@@ -94,32 +94,6 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
return verdict;
}
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
const char *nom2, LLint *size, int *size_flag,
int *depth) {
int depth1 = 0, depth2 = 0;
int flag1 = 0, flag2 = 0;
LLint sz1 = 0, sz2 = 0;
int jok1, jok2, use1;
if (size) {
sz1 = *size;
sz2 = *size;
}
jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
size_flag ? &flag1 : NULL, &depth1);
jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
size_flag ? &flag2 : NULL, &depth2);
use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
if (size)
*size = use1 ? sz1 : sz2;
if (size_flag)
*size_flag = use1 ? flag1 : flag2;
if (depth)
*depth = use1 ? depth1 : depth2;
return use1 ? jok1 : jok2;
}
// supercomparateur joker (tm)
// compare a et b (b=avec joker dedans), case insensitive [voir CI]
// renvoi l'adresse de la première lettre de la chaine

View File

@@ -41,11 +41,6 @@ Please visit our Website: http://www.httrack.com
int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
int *size_flag, int *depth);
/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
scheme); the match latest in the list wins, a "don't know" verdict defers.
Returns the merged verdict; the out-params carry the winner's values. */
int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
const char *nom2, LLint *size, int *size_flag, int *depth);
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
int *size_flag);
const char *strjokerfind(const char *chaine, const char *joker);

View File

@@ -543,7 +543,6 @@ void help(const char *app, int more) {
infomsg("Spider options:");
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
infomsg(" %K load extra cookies from a Netscape cookies.txt");
infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit");
infomsg
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
infomsg

View File

@@ -6006,7 +6006,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
StringCopy(opt->strip_query, "");
StringCopy(opt->cookies_file, "");
StringCopy(opt->why_url, "");
opt->pause_min_ms = 0;
opt->pause_max_ms = 0;
opt->ftp_proxy = HTS_TRUE;
@@ -6154,7 +6153,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
StringFree(opt->mod_blacklist);
StringFree(opt->strip_query);
StringFree(opt->cookies_file);
StringFree(opt->why_url);
StringFree(opt->path_html);
StringFree(opt->path_html_utf8);

View File

@@ -536,8 +536,6 @@ struct httrackp {
(--cookies-file) */
int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
int pause_max_ms; /**< inter-file pause upper bound, ms */
String why_url; /**< URL to diagnose (--why): print the deciding filter rule
and exit without crawling */
};
/* Running statistics for a mirror. */

View File

@@ -620,26 +620,6 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
return 0;
}
/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
static int st_filterdual(httrackp *opt, int argc, char **argv) {
int depth = -1, verdict;
(void) opt;
if (argc < 3) {
fprintf(stderr,
"filterdual: needs <string1> <string2> <filter> [filter...]\n");
return 1;
}
verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
NULL, &depth);
printf("verdict=%s rule=%d\n",
verdict > 0 ? "allowed"
: verdict < 0 ? "forbidden"
: "unknown",
depth);
return 0;
}
static int st_simplify(httrackp *opt, int argc, char **argv) {
(void) opt;
if (argc < 1) {
@@ -2154,8 +2134,6 @@ static const struct selftest_entry {
{"filtersize", "<size> <string> <filter>...",
"size-aware filter verdict (negative size = unknown/scan time)",
st_filtersize},
{"filterdual", "<string1> <string2> <filter>...",
"merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
{"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
{"stripquery", "", "--strip-query pattern/key stripping self-test",
st_stripquery},

View File

@@ -524,11 +524,28 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
// filters, 0=sait pas 1=ok -1=interdit
{
int jokDepth = 0;
int jokDepth1 = 0, jokDepth2 = 0;
int jok1 = 0, jok2 = 0;
jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
NULL, NULL, &jokDepth);
mdepth = _FILTERS[jokDepth];
jok1 =
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
&jokDepth1);
jok2 =
fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
&jokDepth2);
if (jok2 == 0) { // #2 doesn't know
jok = jok1; // then, use #1
mdepth = _FILTERS[jokDepth1];
} else if (jok1 == 0) { // #1 doesn't know
jok = jok2; // then, use #2
mdepth = _FILTERS[jokDepth2];
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
jok = jok1;
mdepth = _FILTERS[jokDepth1];
} else { // #2 matching rule is "after" #1, then it is prioritary
jok = jok2;
mdepth = _FILTERS[jokDepth2];
}
}
if (jok == 1) { // autorisé
@@ -948,10 +965,34 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
// filters, 0=sait pas 1=ok -1=interdit
{
sz = size;
jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
*opt->filters.filptr, lfull, l, &sz, &size_flag,
NULL);
int jokDepth1 = 0, jokDepth2 = 0;
int jok1 = 0, jok2 = 0;
LLint sz1 = size, sz2 = size;
int size_flag1 = 0, size_flag2 = 0;
jok1 =
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
lfull, &sz1, &size_flag1, &jokDepth1);
jok2 =
fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
l, &sz2, &size_flag2, &jokDepth2);
if (jok2 == 0) { // #2 doesn't know
jok = jok1; // then, use #1
sz = sz1;
size_flag = size_flag1;
} else if (jok1 == 0) { // #1 doesn't know
jok = jok2; // then, use #2
sz = sz2;
size_flag = size_flag2;
} else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary
jok = jok1;
sz = sz1;
size_flag = size_flag1;
} else { // #2 matching rule is "after" #1, then it is prioritary
jok = jok2;
sz = sz2;
size_flag = size_flag2;
}
}
// log

View File

@@ -1,20 +0,0 @@
#!/bin/bash
#
set -euo pipefail
# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
# the match latest in the filter list wins, an unknown verdict defers.
fdual() {
local want="$1"
shift
test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
}
fdual 'verdict=unknown rule=0' zzz yyy '+a*' # neither form matches
fdual 'verdict=forbidden rule=0' aaa zzz '-a*' # only form 1 matches
fdual 'verdict=forbidden rule=0' zzz aaa '-a*' # only form 2 matches
fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
fdual 'verdict=allowed rule=0' a aa '+a*' # tie: form 1 wins

View File

@@ -1,35 +0,0 @@
#!/bin/bash
#
set -euo pipefail
# --why: report which +/- filter rule decides for a URL, without crawling.
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' EXIT
why() {
local want="$1" out
shift
out="$(httrack -O "$tmpdir/p" -q "$@" |
grep -E 'accepted|rejected|no filter|unable')"
test "$out" == "$want" || {
echo "want: $want"
echo "got : $out"
exit 1
}
}
base=http://www.example.com
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
--why "$base/x.zip" "$base/" '+*.png' '-*.zip'
why "$base/a.png: accepted by rule #1 (+*.png)" \
--why "$base/a.png" "$base/" '+*.png' '-*.zip'
why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
--why "$base/i.html" "$base/" '+*.png' '-*.zip'
# scheme-less input gets the same http:// default as primary URLs
why "$base/x.zip: rejected by rule #2 (-*.zip)" \
--why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
# the rule latest in the list decides
why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
--why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'

View File

@@ -35,7 +35,6 @@ TESTS = \
01_engine-entities.test \
01_engine-filelist.test \
01_engine-filter.test \
01_engine-filterdual.test \
01_engine-ftp-line.test \
01_engine-ftp-userpass.test \
01_engine-hashtable.test \
@@ -106,7 +105,6 @@ TESTS = \
36_local-bigcrawl.test \
37_local-cache-outage.test \
38_local-update-304.test \
39_local-delayed-cancel.test \
40_local-why.test
39_local-delayed-cancel.test
CLEANFILES = check-network_sh.cache