mirror of
https://github.com/xroche/httrack.git
synced 2026-06-29 13:35:17 +03:00
Compare commits
1 Commits
html5-reso
...
debian-3.4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
141d065a04 |
9
debian/changelog
vendored
9
debian/changelog
vendored
@@ -1,3 +1,12 @@
|
||||
httrack (3.49.10-2) unstable; urgency=medium
|
||||
|
||||
* Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
|
||||
absent (the local-server tests need python3, which the buildds lack). Add
|
||||
patches/skip-local-pause-test-without-python3.patch to guard the test on
|
||||
python3 up front, like its siblings, so it skips cleanly.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 28 Jun 2026 20:18:46 +0200
|
||||
|
||||
httrack (3.49.10-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: new download-pacing and URL-handling options plus a
|
||||
|
||||
1
debian/patches/series
vendored
Normal file
1
debian/patches/series
vendored
Normal file
@@ -0,0 +1 @@
|
||||
skip-local-pause-test-without-python3.patch
|
||||
29
debian/patches/skip-local-pause-test-without-python3.patch
vendored
Normal file
29
debian/patches/skip-local-pause-test-without-python3.patch
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
Description: skip 28_local-pause when python3 is absent (fixes FTBFS on buildds)
|
||||
The local-server tests skip (exit 77) when python3 is missing, but
|
||||
28_local-pause wrapped local-crawl.sh in a command substitution that swallowed
|
||||
that skip. On the python3-less buildd chroot the test then ran serverless, both
|
||||
crawls finished in 0s, and the 0s delta was reported as a failure. Guard the
|
||||
test on python3 up front, like its siblings, so it skips cleanly.
|
||||
Author: Xavier Roche <roche@httrack.com>
|
||||
Origin: upstream, https://github.com/xroche/httrack/pull/445
|
||||
Applied-Upstream: https://github.com/xroche/httrack/commit/45279d7
|
||||
Forwarded: not-needed
|
||||
Last-Update: 2026-06-28
|
||||
diff --git a/tests/28_local-pause.test b/tests/28_local-pause.test
|
||||
index 8505a75..add95d1 100755
|
||||
--- a/tests/28_local-pause.test
|
||||
+++ b/tests/28_local-pause.test
|
||||
@@ -9,6 +9,13 @@ set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
+# python3 runs the local server (mirror local-crawl.sh); skip when absent, else
|
||||
+# run() swallows its exit-77 and the serverless 0s/0s crawl looks like a fail.
|
||||
+command -v python3 >/dev/null || {
|
||||
+ echo "python3 not found; skipping local crawl tests"
|
||||
+ exit 77
|
||||
+}
|
||||
+
|
||||
run() { # echoes the wall-clock seconds of one crawl
|
||||
local t0 t1
|
||||
t0=$(date +%s)
|
||||
@@ -229,10 +229,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_DEFAULT_FOOTER \
|
||||
"<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION \
|
||||
" " HTTRACK_AFF_AUTHORS ", %s -->"
|
||||
/* Honest crawler User-Agent; no fake OS/browser to go stale. */
|
||||
#define HTS_DEFAULT_USER_AGENT \
|
||||
"Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION \
|
||||
"; +https://www.httrack.com/)"
|
||||
#define HTTRACK_WEB "http://www.httrack.com"
|
||||
#define HTS_UPDATE_WEBSITE \
|
||||
"http://www.httrack.com/" \
|
||||
|
||||
82
src/htslib.c
82
src/htslib.c
@@ -563,39 +563,6 @@ const char *hts_mime[][2] = {
|
||||
{"", ""}
|
||||
};
|
||||
|
||||
/* Modern web formats (post-2010), kept in their own table: appending to the
|
||||
legacy hts_mime[] above makes clang-format reflow its whole initializer.
|
||||
Scanned after hts_mime[], so it never shadows a legacy mapping. */
|
||||
static const char *hts_mime_modern[][2] = {
|
||||
{"image/webp", "webp"},
|
||||
{"image/avif", "avif"},
|
||||
{"image/heic", "heic"},
|
||||
{"font/woff", "woff"},
|
||||
{"font/woff2", "woff2"},
|
||||
{"font/ttf", "ttf"},
|
||||
{"font/otf", "otf"},
|
||||
{"application/json", "json"},
|
||||
{"application/ld+json", "jsonld"},
|
||||
{"application/manifest+json", "webmanifest"},
|
||||
{"application/wasm", "wasm"},
|
||||
{"text/javascript", "js"},
|
||||
{"text/javascript", "mjs"},
|
||||
{"text/markdown", "md"},
|
||||
{"video/mp4", "mp4"},
|
||||
{"video/webm", "webm"},
|
||||
{"video/ogg", "ogv"},
|
||||
{"video/mp2t", "ts"},
|
||||
{"audio/mp4", "m4a"},
|
||||
{"audio/aac", "aac"},
|
||||
{"audio/ogg", "oga"},
|
||||
{"audio/opus", "opus"},
|
||||
{"audio/flac", "flac"},
|
||||
{"audio/webm", "weba"},
|
||||
{"application/x-7z-compressed", "7z"},
|
||||
{"application/x-rar-compressed", "rar"},
|
||||
{"application/zstd", "zst"},
|
||||
{"", ""}};
|
||||
|
||||
// Reserved (RFC2396)
|
||||
#define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
|
||||
#define CHAR_RESERVED(c) ( CIS(c,';') \
|
||||
@@ -4341,20 +4308,6 @@ void guess_httptype(httrackp * opt, char *s, const char *fil) {
|
||||
(void) get_httptype_sized(opt, s, HTS_MIMETYPE_SIZE, fil, 1);
|
||||
}
|
||||
|
||||
// first match in a NUL-terminated {mime,ext} table. key selects the lookup
|
||||
// column (0=mime, 1=ext); returns the other column, or NULL if no row matches
|
||||
// (a "*" partner means the row carries no value).
|
||||
static const char *hts_mime_lookup(const char *(*table)[2], int key,
|
||||
const char *needle) {
|
||||
int j;
|
||||
|
||||
for (j = 0; strnotempty(table[j][1]); j++) {
|
||||
if (strfield2(table[j][key], needle) && table[j][!key][0] != '*')
|
||||
return table[j][!key];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// write the mime type for fil into s (capacity ssize)
|
||||
// flag: 1 to always return a type (the "application/..." / octet-stream
|
||||
// fallback) returns 1 if a type was written to s, 0 otherwise
|
||||
@@ -4378,15 +4331,17 @@ HTSEXT_API hts_boolean get_httptype_sized(httrackp *opt, char *s, size_t ssize,
|
||||
while ((a > fil) && (*a != '.') && (*a != '/'))
|
||||
a--;
|
||||
if (a >= fil && *a == '.' && strlen(a) < 32) {
|
||||
const char *mime;
|
||||
int j = 0;
|
||||
|
||||
a++;
|
||||
mime = hts_mime_lookup(hts_mime, 1, a);
|
||||
if (mime == NULL)
|
||||
mime = hts_mime_lookup(hts_mime_modern, 1, a);
|
||||
if (mime != NULL) {
|
||||
strlcpybuff(s, mime, ssize);
|
||||
return 1;
|
||||
while(strnotempty(hts_mime[j][1])) {
|
||||
if (strfield2(hts_mime[j][1], a)) {
|
||||
if (hts_mime[j][0][0] != '*') { // a match exists
|
||||
strlcpybuff(s, hts_mime[j][0], ssize);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
j++;
|
||||
}
|
||||
|
||||
if (flag) {
|
||||
@@ -4521,16 +4476,18 @@ int get_userhttptype(httrackp * opt, char *s, const char *fil) {
|
||||
// returns 1 if an extension was found (and written to s), 0 otherwise
|
||||
int give_mimext(char *s, size_t ssize, const char *st) {
|
||||
int ok = 0;
|
||||
const char *ext;
|
||||
int j = 0;
|
||||
|
||||
st = hts_effective_mime(st); /* no declared type: derive an html ext */
|
||||
s[0] = '\0';
|
||||
ext = hts_mime_lookup(hts_mime, 0, st);
|
||||
if (ext == NULL)
|
||||
ext = hts_mime_lookup(hts_mime_modern, 0, st);
|
||||
if (ext != NULL) {
|
||||
strlcpybuff(s, ext, ssize);
|
||||
ok = 1;
|
||||
while((!ok) && (strnotempty(hts_mime[j][1]))) {
|
||||
if (strfield2(hts_mime[j][0], st)) {
|
||||
if (hts_mime[j][1][0] != '*') { // a match exists
|
||||
strlcpybuff(s, hts_mime[j][1], ssize);
|
||||
ok = 1;
|
||||
}
|
||||
}
|
||||
j++;
|
||||
}
|
||||
// wrap "x" mimetypes, such as:
|
||||
// application/x-mp3
|
||||
@@ -6048,7 +6005,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
opt->shell = HTS_FALSE;
|
||||
opt->proxy.active = 0; // pas de proxy
|
||||
opt->user_agent_send = HTS_TRUE;
|
||||
StringCopy(opt->user_agent, HTS_DEFAULT_USER_AGENT);
|
||||
StringCopy(opt->user_agent,
|
||||
"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)");
|
||||
StringCopy(opt->referer, "");
|
||||
StringCopy(opt->from, "");
|
||||
opt->savename_83 = HTS_SAVENAME_83_LONG; // long names by default
|
||||
|
||||
@@ -239,14 +239,6 @@ static void basic_selftests(void) {
|
||||
assertf(strcmp(ext, "html") == 0);
|
||||
assertf(give_mimext(ext, sizeof(ext), "no/such-mime-type") == 0);
|
||||
assertf(ext[0] == '\0');
|
||||
// modern web formats -> extension. Avoid MIME types the
|
||||
// application/<=4-char-subtype fallback could fabricate without a row.
|
||||
assertf(give_mimext(ext, sizeof(ext), "image/webp") == 1);
|
||||
assertf(strcmp(ext, "webp") == 0);
|
||||
assertf(give_mimext(ext, sizeof(ext), "application/manifest+json") == 1);
|
||||
assertf(strcmp(ext, "webmanifest") == 0);
|
||||
assertf(give_mimext(ext, sizeof(ext), "font/woff2") == 1);
|
||||
assertf(strcmp(ext, "woff2") == 0);
|
||||
}
|
||||
// convtolower(): lower-cases into the caller buffer (bounded by its size).
|
||||
{
|
||||
@@ -301,16 +293,6 @@ static void basic_selftests(void) {
|
||||
assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
|
||||
"x.gif", 0) == 1);
|
||||
assertf(strcmp(r.contenttype, "image/gif") == 0);
|
||||
// modern extensions map back to their MIME type
|
||||
assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
|
||||
"x.webp", 0) == 1);
|
||||
assertf(strcmp(r.contenttype, "image/webp") == 0);
|
||||
assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
|
||||
"app.wasm", 0) == 1);
|
||||
assertf(strcmp(r.contenttype, "application/wasm") == 0);
|
||||
assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
|
||||
"mod.mjs", 0) == 1);
|
||||
assertf(strcmp(r.contenttype, "text/javascript") == 0);
|
||||
// no extension and flag==0: nothing written, returns 0
|
||||
assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
|
||||
"noextfile", 0) == 0);
|
||||
@@ -1302,22 +1284,6 @@ static int st_urlhack(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Default User-Agent: honest HTTrack token, no resurrected Windows 98. */
|
||||
static int st_useragent(httrackp *opt, int argc, char **argv) {
|
||||
const char *ua = StringBuff(opt->user_agent);
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
assertf(ua != NULL);
|
||||
assertf(strcmp(ua, HTS_DEFAULT_USER_AGENT) == 0);
|
||||
/* Teeth independent of the macro: honest token + self-identifier, and no
|
||||
legacy Mozilla/4.x fake-browser string (rejects the whole relic family). */
|
||||
assertf(strstr(ua, "HTTrack/") != NULL);
|
||||
assertf(strstr(ua, "+https://www.httrack.com/") != NULL);
|
||||
assertf(strstr(ua, "Mozilla/4.") == NULL);
|
||||
printf("useragent self-test OK: %s\n", ua);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* Registry: name -> handler, with a usage hint and a one-line description. */
|
||||
/* ------------------------------------------------------------ */
|
||||
@@ -1364,7 +1330,6 @@ static const struct selftest_entry {
|
||||
st_cache_writefail},
|
||||
{"dns", "", "DNS resolver/cache self-test", st_dns},
|
||||
{"cookies", "", "cookie request-header self-test", st_cookies},
|
||||
{"useragent", "", "default User-Agent self-test", st_useragent},
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
@@ -358,12 +358,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
{NULL, 0}
|
||||
};
|
||||
initStrElt initStr[] = {
|
||||
{"user", HTS_DEFAULT_USER_AGENT},
|
||||
{"footer", "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x "
|
||||
"[XR&CO'2014], %s -->"},
|
||||
{"url2",
|
||||
"+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
|
||||
{NULL, NULL}};
|
||||
{"user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"},
|
||||
{"footer",
|
||||
"<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2014], %s -->"},
|
||||
{"url2", "+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
|
||||
{NULL, NULL}
|
||||
};
|
||||
int i = 0;
|
||||
|
||||
for(i = 0; initInt[i].name; i++) {
|
||||
|
||||
@@ -80,10 +80,6 @@ htspair_t hts_detect_embed[] = {
|
||||
{NULL, NULL}
|
||||
};
|
||||
|
||||
/* HTML5 media siblings of <img src>: same near-link treatment (#451) */
|
||||
static const htspair_t hts_detect_embed_html5[] = {
|
||||
{"source", "src"}, {"source", "srcset"}, {"track", "src"}, {NULL, NULL}};
|
||||
|
||||
/* Internal */
|
||||
static int hts_acceptlink_(httrackp * opt, int ptr, const char *adr,
|
||||
const char *fil, const char *tag,
|
||||
@@ -140,17 +136,6 @@ static int cmp_token(const char *tag, const char *cmp) {
|
||||
&& !isalnum((unsigned char) tag[p]));
|
||||
}
|
||||
|
||||
/* TRUE if (tag, attribute) matches an embedded-asset pair in the table */
|
||||
static hts_boolean is_embed_pair(const htspair_t *table, const char *tag,
|
||||
const char *attribute) {
|
||||
int i;
|
||||
for (i = 0; table[i].tag != NULL; i++) {
|
||||
if (cmp_token(tag, table[i].tag) && cmp_token(attribute, table[i].attr))
|
||||
return HTS_TRUE;
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
const char *adr, const char *fil, const char *tag,
|
||||
const char *attribute, int *set_prio_to,
|
||||
@@ -178,9 +163,15 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
/* Built-in known tags (<img src=..>, ..) */
|
||||
if (forbidden_url != 0 && opt->nearlink && tag != NULL && attribute != NULL) {
|
||||
if (is_embed_pair(hts_detect_embed, tag, attribute) ||
|
||||
is_embed_pair(hts_detect_embed_html5, tag, attribute)) {
|
||||
embedded_triggered = 1;
|
||||
int i;
|
||||
|
||||
for(i = 0; hts_detect_embed[i].tag != NULL; i++) {
|
||||
if (cmp_token(tag, hts_detect_embed[i].tag)
|
||||
&& cmp_token(attribute, hts_detect_embed[i].attr)
|
||||
) {
|
||||
embedded_triggered = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -323,33 +323,4 @@ grep -Fq 'href="ahref%20(4).gif"' "$saved9" ||
|
||||
! grep -Eq '(src|href)="[^"]*%28' "$saved9" ||
|
||||
! echo "FAIL #163: gate over-fired onto a non-url() attribute link" || exit 1
|
||||
|
||||
# HTML5 <source>/<track> follow as embedded near-links past the -r2 depth boundary (#451).
|
||||
# img.gif positive control; plain.gif (bare <a href>) negative control proves the gate is selective.
|
||||
site10="$tmp/html5media"
|
||||
mkdir -p "$site10"
|
||||
for f in img ss plain; do gif "$site10/$f.gif"; done
|
||||
printf 'x' >"$site10/v.webm"
|
||||
printf 'x' >"$site10/subs.vtt"
|
||||
cat >"$site10/index.html" <<EOF
|
||||
<html><body><a href="leaf.html">leaf</a></body></html>
|
||||
EOF
|
||||
cat >"$site10/leaf.html" <<EOF
|
||||
<html><body>
|
||||
<img src="img.gif">
|
||||
<picture><source srcset="ss.gif 2x"></picture>
|
||||
<video><source src="v.webm"></video>
|
||||
<video><track src="subs.vtt"></video>
|
||||
<a href="plain.gif">plain link past the boundary</a>
|
||||
</body></html>
|
||||
EOF
|
||||
out10="$tmp/html5media-out"
|
||||
rm -rf "$out10"
|
||||
mkdir -p "$out10"
|
||||
httrack "file://$site10/index.html" -O "$out10" --quiet --near -r2 >"$out10/.log" 2>&1 || true
|
||||
found "img.gif" "$out10"
|
||||
found "ss.gif" "$out10"
|
||||
found "v.webm" "$out10"
|
||||
found "subs.vtt" "$out10"
|
||||
notfound "plain.gif" "$out10"
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Default User-Agent (#449): honest HTTrack token, no Windows 98 relic.
|
||||
httrack -O /dev/null -#test=useragent run | grep -q "useragent self-test OK"
|
||||
@@ -50,7 +50,6 @@ TESTS = \
|
||||
01_engine-stripquery.test \
|
||||
01_engine-strsafe.test \
|
||||
01_engine-urlhack.test \
|
||||
01_engine-useragent.test \
|
||||
02_manpage-regen.test \
|
||||
02_update-cache.test \
|
||||
10_crawl-simple.test \
|
||||
|
||||
Reference in New Issue
Block a user