mirror of
https://github.com/xroche/httrack.git
synced 2026-07-05 16:44:55 +03:00
Compare commits
1 Commits
p2-xsize-h
...
string-mac
| Author | SHA1 | Date | |
|---|---|---|---|
|
02796dff34 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -39,6 +39,3 @@ Makefile
|
||||
|
||||
# Editor / autotools backup files.
|
||||
*~
|
||||
|
||||
# Python bytecode (tests/local-server.py).
|
||||
__pycache__/
|
||||
|
||||
19
configure.ac
19
configure.ac
@@ -1,6 +1,6 @@
|
||||
AC_PREREQ([2.71])
|
||||
|
||||
AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_INIT([httrack], [3.49.10], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_COPYRIGHT([
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2015 Xavier Roche and other contributors
|
||||
@@ -29,11 +29,10 @@ AC_CONFIG_SRCDIR(src/httrack.c)
|
||||
AC_CONFIG_MACRO_DIR([m4])
|
||||
AC_CONFIG_HEADERS(config.h)
|
||||
AM_INIT_AUTOMAKE([subdir-objects])
|
||||
# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
|
||||
# installed headers (no struct layout or exported signature changed vs
|
||||
# 3.49.10), so it stays soname .so.3; bump revision.
|
||||
# 3:2:0: 3.49.10 only appends tail fields to the options struct (no existing
|
||||
# symbol or offset changed vs 3.49.9), so it stays soname .so.3; bump revision.
|
||||
# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
|
||||
VERSION_INFO="3:3:0"
|
||||
VERSION_INFO="3:2:0"
|
||||
AM_MAINTAINER_MODE
|
||||
AC_USE_SYSTEM_EXTENSIONS
|
||||
|
||||
@@ -64,16 +63,6 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
|
||||
# Export version info
|
||||
AC_SUBST(VERSION_INFO)
|
||||
|
||||
# Versioned plugin name for dlopen() in hts_create_opt(); soname major is
|
||||
# libtool's current - age, so this tracks VERSION_INFO bumps automatically.
|
||||
HTS_SONAME_MAJOR=$((${VERSION_INFO%%:*} - ${VERSION_INFO##*:}))
|
||||
case "$host_os" in
|
||||
darwin*) HTS_LIBHTSJAVA_NAME="libhtsjava.$HTS_SONAME_MAJOR.dylib" ;;
|
||||
*) HTS_LIBHTSJAVA_NAME="libhtsjava.so.$HTS_SONAME_MAJOR" ;;
|
||||
esac
|
||||
AC_DEFINE_UNQUOTED([HTS_LIBHTSJAVA_NAME], ["$HTS_LIBHTSJAVA_NAME"],
|
||||
[Versioned libhtsjava runtime name, derived from VERSION_INFO])
|
||||
|
||||
### Default CFLAGS
|
||||
DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
|
||||
-Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
|
||||
|
||||
21
debian/changelog
vendored
21
debian/changelog
vendored
@@ -1,24 +1,3 @@
|
||||
httrack (3.49.11-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: crawl correctness and security fixes (network-facing
|
||||
buffer overflows, file-type detection, redirect handling) and modernized
|
||||
web defaults; full list in history.txt.
|
||||
* Add DEP-12 upstream metadata (#466).
|
||||
* Bump debhelper compat to 14 (#466).
|
||||
* Drop the redundant Priority field and update the NMU lintian override to
|
||||
the current tag names (#466).
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 05 Jul 2026 00:03:18 +0200
|
||||
|
||||
httrack (3.49.10-2) unstable; urgency=medium
|
||||
|
||||
* Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
|
||||
absent (the local-server tests need python3, which the buildds lack). Add
|
||||
patches/skip-local-pause-test-without-python3.patch to guard the test on
|
||||
python3 up front, like its siblings, so it skips cleanly.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 28 Jun 2026 20:18:46 +0200
|
||||
|
||||
httrack (3.49.10-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: new download-pacing and URL-handling options plus a
|
||||
|
||||
3
debian/control
vendored
3
debian/control
vendored
@@ -1,8 +1,9 @@
|
||||
Source: httrack
|
||||
Section: web
|
||||
Priority: optional
|
||||
Maintainer: Xavier Roche <roche@httrack.com>
|
||||
Standards-Version: 4.7.4
|
||||
Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
|
||||
Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
|
||||
Rules-Requires-Root: no
|
||||
Homepage: http://www.httrack.com
|
||||
Vcs-Git: https://github.com/xroche/httrack.git
|
||||
|
||||
4
debian/source/lintian-overrides
vendored
4
debian/source/lintian-overrides
vendored
@@ -1,6 +1,4 @@
|
||||
# Maintainer uploads sign the changelog as xavier@debian.org while the control
|
||||
# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
|
||||
httrack source: no-nmu-in-changelog
|
||||
httrack source: changelog-should-mention-nmu
|
||||
httrack source: source-nmu-has-incorrect-version-number
|
||||
|
||||
# The bundled HTML pages are the genuine upstream documentation taken from
|
||||
|
||||
6
debian/upstream/metadata
vendored
6
debian/upstream/metadata
vendored
@@ -1,6 +0,0 @@
|
||||
---
|
||||
Repository: https://github.com/xroche/httrack.git
|
||||
Repository-Browse: https://github.com/xroche/httrack
|
||||
Bug-Database: https://github.com/xroche/httrack/issues
|
||||
Bug-Submit: https://github.com/xroche/httrack/issues/new
|
||||
Contact: Xavier Roche <roche@httrack.com>
|
||||
17
history.txt
17
history.txt
@@ -4,23 +4,6 @@ HTTrack Website Copier release history:
|
||||
|
||||
This file lists all changes and fixes that have been made for HTTrack
|
||||
|
||||
3.49-11
|
||||
+ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
|
||||
+ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
|
||||
+ New: follow <source> and <track> media elements as embedded links (#451)
|
||||
+ New: added modern web MIME types to the type/extension table (#448)
|
||||
+ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
|
||||
+ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
|
||||
+ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
|
||||
+ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
|
||||
+ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
|
||||
+ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
|
||||
+ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
|
||||
+ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
|
||||
+ Changed: decode the full WHATWG set of HTML named character references (#443)
|
||||
+ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
|
||||
+ Changed: multiple internal hardening, build, test and CI improvements
|
||||
|
||||
3.49-10
|
||||
+ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
|
||||
+ New: --pause to space out file downloads by a random delay (#185)
|
||||
|
||||
@@ -62,7 +62,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
|
||||
@@ -70,7 +70,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
|
||||
htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
|
||||
htshelp.h htsindex.h htslib.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsmodules.h htsname.h htsnet.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
|
||||
120
src/htsback.c
120
src/htsback.c
@@ -572,12 +572,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
&& back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
|
||||
if (back[p].status == STATUS_READY) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"incomplete transfer (expected " LLintP
|
||||
" bytes, got " LLintP
|
||||
"): file not cached, will be retried on the next update"
|
||||
" (use -%%B to cache anyway): %s%s",
|
||||
back[p].r.totalsize, back[p].r.size, back[p].url_adr,
|
||||
back[p].url_fil);
|
||||
"file not stored in cache due to bogus state (broken size, expected "
|
||||
LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
|
||||
back[p].r.size, back[p].url_adr, back[p].url_fil);
|
||||
} else {
|
||||
hts_log_print(opt, LOG_INFO,
|
||||
"incomplete file not yet stored in cache (expected "
|
||||
@@ -882,12 +879,11 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
back[p].url_fil, NULL);
|
||||
} else {
|
||||
/* Partial file, but marked as "ok" ? */
|
||||
hts_log_print(
|
||||
opt, LOG_WARNING,
|
||||
"file with unresolved type not cached (%s (%d), size " LLintP
|
||||
"): %s%s",
|
||||
back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
|
||||
back[p].url_adr, back[p].url_fil);
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"file not stored in cache due to bogus state (incomplete type with %s (%d), size "
|
||||
LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
|
||||
(LLint) back[p].r.size, back[p].url_adr,
|
||||
back[p].url_fil);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1363,18 +1359,6 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
}
|
||||
|
||||
// effacer entrée
|
||||
/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
|
||||
static void back_delayed_discard(httrackp *opt, lien_back *back) {
|
||||
if (back->r.out != NULL) {
|
||||
fclose(back->r.out);
|
||||
back->r.out = NULL;
|
||||
}
|
||||
back->r.is_write = 0;
|
||||
if (opt != NULL)
|
||||
url_savename_refname_remove(opt, back->url_adr, back->url_fil);
|
||||
(void) UNLINK(back->url_sav);
|
||||
}
|
||||
|
||||
int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p) {
|
||||
lien_back *const back = sback->lnk;
|
||||
@@ -1382,12 +1366,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
|
||||
assertf(p >= 0 && p < back_max);
|
||||
if (p >= 0 && p < sback->count) { // on sait jamais..
|
||||
/* mid-write cancel: drop a .delayed placeholder; real-named partials
|
||||
survive for resume (--continue) */
|
||||
if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
|
||||
(back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
|
||||
back_delayed_discard(opt, &back[p]);
|
||||
}
|
||||
// Vérificateur d'intégrité
|
||||
#if DEBUG_CHECKINT
|
||||
_CHECKINT(&back[p], "Appel back_delete")
|
||||
@@ -2259,13 +2237,12 @@ int host_wait(httrackp * opt, lien_back * back) {
|
||||
|
||||
static int slot_can_be_cleaned(const lien_back * back) {
|
||||
return (back->status == STATUS_READY) // ready
|
||||
/* Check autoclean */
|
||||
&& (!back->locked) // not held by hts_wait_delayed (name pending)
|
||||
&& (!back->testmode) // not test mode
|
||||
&& (strnotempty(back->url_sav)) // filename exists
|
||||
&& (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
|
||||
&& (back->r.size >= 0) // size>=0
|
||||
;
|
||||
/* Check autoclean */
|
||||
&& (!back->testmode) // not test mode
|
||||
&& (strnotempty(back->url_sav)) // filename exists
|
||||
&& (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
|
||||
&& (back->r.size >= 0) // size>=0
|
||||
;
|
||||
}
|
||||
|
||||
static int slot_can_be_finalized(httrackp * opt, const lien_back * back) {
|
||||
@@ -2441,34 +2418,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back_clean(opt, cache, sback);
|
||||
#endif
|
||||
|
||||
/* Time limit exceeded past grace: abort in-flight transfers so no wait loop
|
||||
starves (#481). FTP slots stay, their thread owns the socket. */
|
||||
if (!back_checkmirror(opt)) {
|
||||
int aborted = 0;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < (unsigned int) back_max; i++) {
|
||||
if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
|
||||
if (back[i].r.soc != INVALID_SOCKET) {
|
||||
deletehttp(&back[i].r);
|
||||
}
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
/* drop a .delayed placeholder; real partials survive for resume */
|
||||
if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
|
||||
back_delayed_discard(opt, &back[i]);
|
||||
back[i].r.statuscode = STATUSCODE_TIMEOUT;
|
||||
strcpybuff(back[i].r.msg, "Mirror Time Out");
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
aborted++;
|
||||
}
|
||||
}
|
||||
if (aborted > 0)
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"time limit reached, %d transfer(s) aborted", aborted);
|
||||
return;
|
||||
}
|
||||
|
||||
// recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
|
||||
do_wait = 0;
|
||||
gestion_timeout = 0;
|
||||
@@ -2942,10 +2891,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// range size hack old location
|
||||
|
||||
#if HTS_DIRECTDISK
|
||||
// Shortcut: store the file directly on disk when possible,
|
||||
// sparing memory
|
||||
if (back[i].status &&
|
||||
!back[i].locked) { // name still pending when locked
|
||||
// Court-circuit:
|
||||
// Peut-on stocker le fichier directement sur disque?
|
||||
// Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci!
|
||||
if (back[i].status) {
|
||||
if (back[i].r.is_write == 0) { // mode mémoire
|
||||
if (back[i].r.adr == NULL) { // rien n'a été écrit
|
||||
if (!back[i].testmode) { // pas mode test
|
||||
@@ -4011,12 +3960,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
&& (back[i].r.adr = (char *) malloct(2))) {
|
||||
back[i].r.adr[0] = 0;
|
||||
}
|
||||
/* locked = name pending; the waiter finalizes after
|
||||
patching url_sav (else: cached as .delayed, #5) */
|
||||
if (!back[i].locked) {
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing empty");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
}
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing empty");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
} else if (!back[i].r.is_chunk) { // pas de chunk
|
||||
//if (back[i].r.http11!=2) { // pas de chunk
|
||||
back[i].is_chunk = 0;
|
||||
@@ -4214,11 +4159,6 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Grace left to the smooth stop before in-flight transfers are aborted. */
|
||||
static int back_maxtime_grace(const int maxtime) {
|
||||
return maximum(5, minimum(30, maxtime / 10));
|
||||
}
|
||||
|
||||
int back_checkmirror(httrackp * opt) {
|
||||
// Check max size
|
||||
if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
|
||||
@@ -4235,19 +4175,13 @@ int back_checkmirror(httrackp * opt) {
|
||||
*/
|
||||
}
|
||||
// Check max time
|
||||
if (opt->maxtime > 0) {
|
||||
const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
|
||||
|
||||
if (elapsed >= opt->maxtime) {
|
||||
if (!opt->state.stop) { /* not yet stopped */
|
||||
hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
|
||||
opt->maxtime);
|
||||
/* cancel mirror smoothly */
|
||||
hts_request_stop(opt, 0);
|
||||
}
|
||||
/* smooth stop starved past the grace period: stop waiting (#481) */
|
||||
if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
|
||||
return 0;
|
||||
if ((opt->maxtime > 0)
|
||||
&& ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
|
||||
if (!opt->state.stop) { /* not yet stopped */
|
||||
hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
|
||||
opt->maxtime);
|
||||
/* cancel mirror smoothly */
|
||||
hts_request_stop(opt, 0);
|
||||
}
|
||||
}
|
||||
return 1; /* Ok, go on */
|
||||
|
||||
@@ -136,8 +136,6 @@ void back_solve(httrackp * opt, lien_back * sback);
|
||||
int host_wait(httrackp * opt, lien_back * sback);
|
||||
#endif
|
||||
int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
|
||||
/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
|
||||
the -E deadline overran its grace period (callers must stop waiting). */
|
||||
int back_checkmirror(httrackp * opt);
|
||||
|
||||
#endif
|
||||
|
||||
185
src/htscache.c
185
src/htscache.c
@@ -40,7 +40,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htsbasenet.h"
|
||||
#include "htsmd5.h"
|
||||
#include <limits.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "htszlib.h"
|
||||
@@ -221,38 +220,23 @@ struct cache_back_zip_entry {
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* Consecutive entry write failures before the cache stream is declared dead. */
|
||||
#define CACHE_MAX_WRITE_FAILURES 8
|
||||
|
||||
/* Cache write failed: a fatal errno or a failure streak aborts the mirror
|
||||
(exit_xh); an isolated failure only drops the current entry. */
|
||||
/* A cache (new.zip) write failed: storage is gone (disk full / dropped share),
|
||||
so the mirror is doomed too. Abort it via exit_xh, don't crash as assertf
|
||||
did. */
|
||||
static void cache_zip_write_failed(httrackp *opt, cache_back *cache,
|
||||
const char *what, int zErr,
|
||||
hts_boolean entry_open, const char *url_adr,
|
||||
const char *url_fil) {
|
||||
const int fatal_errno = zErr == ZIP_ERRNO && check_fatal_io_errno();
|
||||
|
||||
cache->zipWriteFailures++;
|
||||
if (fatal_errno || cache->zipWriteFailures >= CACHE_MAX_WRITE_FAILURES) {
|
||||
if (!cache->zipWriteFailed) {
|
||||
cache->zipWriteFailed = HTS_TRUE;
|
||||
if (fatal_errno) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: disk full or filesystem problems");
|
||||
} else {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: cache write failed (%s): %s", what,
|
||||
hts_get_zerror(zErr));
|
||||
}
|
||||
const char *what, int zErr) {
|
||||
if (!cache->zipWriteFailed) {
|
||||
cache->zipWriteFailed = HTS_TRUE;
|
||||
if (check_fatal_io_errno()) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: disk full or filesystem problems");
|
||||
} else {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: cache write failed (%s): %s", what,
|
||||
hts_get_zerror(zErr));
|
||||
}
|
||||
opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
|
||||
} else {
|
||||
if (entry_open)
|
||||
zipCloseFileInZip((zipFile) cache->zipOutput); /* abandon, best-effort */
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"cache write failed (%s: %s), entry not cached: %s%s", what,
|
||||
hts_get_zerror(zErr), url_adr, url_fil);
|
||||
}
|
||||
opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
|
||||
}
|
||||
|
||||
/* Ajout d'un fichier en cache */
|
||||
@@ -302,19 +286,10 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if (r->size < 0) // error
|
||||
return;
|
||||
|
||||
// data in cache: the body must fit the 32-bit zip write API
|
||||
if (dataincache && (LLint) (int) r->size != r->size) {
|
||||
if (r->is_write && url_save != NULL && strnotempty(url_save)) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"file too large for the cache, storing headers only: %s%s",
|
||||
url_adr, url_fil);
|
||||
dataincache = 0;
|
||||
} else {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"entry too large for the cache, not cached: %s%s", url_adr,
|
||||
url_fil);
|
||||
return;
|
||||
}
|
||||
// data in cache
|
||||
if (dataincache) {
|
||||
assertf(((int) r->size) == r->size);
|
||||
//entryBodySize = (int) r->size;
|
||||
}
|
||||
|
||||
/* Fields */
|
||||
@@ -394,8 +369,7 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
*/
|
||||
headers, (uInt) strlen(headers), NULL, 0, NULL, /* comment */
|
||||
Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "opening a cache entry", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
cache_zip_write_failed(opt, cache, "opening a cache entry", zErr);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -406,8 +380,7 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if ((zErr =
|
||||
zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr,
|
||||
(int) r->size)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
|
||||
HTS_TRUE, url_adr, url_fil);
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -429,8 +402,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if ((zErr =
|
||||
zipWriteInFileInZip((zipFile) cache->zipOutput, buff,
|
||||
(int) nl)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
|
||||
HTS_TRUE, url_adr, url_fil);
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache",
|
||||
zErr);
|
||||
fclose(fp);
|
||||
return;
|
||||
}
|
||||
@@ -446,19 +419,15 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
|
||||
/* Close */
|
||||
if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "closing a cache entry", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
cache_zip_write_failed(opt, cache, "closing a cache entry", zErr);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Flush */
|
||||
if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) {
|
||||
cache_zip_write_failed(opt, cache, "flushing the cache", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
cache_zip_write_failed(opt, cache, "flushing the cache", zErr);
|
||||
return;
|
||||
}
|
||||
|
||||
cache->zipWriteFailures = 0; /* entry stored: reset the failure streak */
|
||||
}
|
||||
|
||||
#else
|
||||
@@ -627,18 +596,15 @@ htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
|
||||
return cache_readex(opt, cache, adr, fil, save, location, NULL, 1);
|
||||
}
|
||||
|
||||
htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
|
||||
const char *adr, const char *fil,
|
||||
char *return_save) {
|
||||
htsblk r = cache_readex(opt, cache, adr, fil, NULL, NULL, return_save, 0);
|
||||
htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
|
||||
const char *adr, const char *fil) {
|
||||
htsblk r = cache_read(opt, cache, adr, fil, NULL, NULL);
|
||||
|
||||
if (r.statuscode == -1) {
|
||||
lien_back *itemback = NULL;
|
||||
|
||||
if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
|
||||
r = itemback->r;
|
||||
if (return_save != NULL)
|
||||
strlcpybuff(return_save, itemback->url_sav, HTS_URLMAXSIZE * 2);
|
||||
/* cleanup */
|
||||
back_clear_entry(itemback); /* delete entry content */
|
||||
freet(itemback); /* delete item */
|
||||
@@ -799,15 +765,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
|
||||
}
|
||||
|
||||
/* A negative X-Size is corrupt; so is one >= INT_MAX when the data
|
||||
is in the zip (the write path asserts int-sized). Headers-only
|
||||
entries legitimately exceed INT_MAX (>2GB body on disk): keep
|
||||
them, or every update would re-fetch the file. */
|
||||
if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Read Error : Bad Size");
|
||||
}
|
||||
|
||||
/* Complete fields */
|
||||
r.totalsize = r.size;
|
||||
r.adr = NULL;
|
||||
@@ -834,8 +791,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
} // otherwise, the ZIP file is supposed to be consistent with data.
|
||||
}
|
||||
/* Read data ? */
|
||||
else if (r.statuscode !=
|
||||
STATUSCODE_INVALID) { /* ne pas lire uniquement header */
|
||||
else { /* ne pas lire uniquement header */
|
||||
int ok = 0;
|
||||
|
||||
#if HTS_DIRECTDISK
|
||||
@@ -999,10 +955,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strcpybuff(r.msg,
|
||||
"Previous cache file not found (empty filename)");
|
||||
}
|
||||
} else if (r.size >= INT_MAX) { /* too big to read in memory */
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Read Error : Bad Size");
|
||||
} else { /* Read in memory from disk */
|
||||
} else { /* Read in memory from disk */
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
|
||||
|
||||
if (fp != NULL) {
|
||||
@@ -1464,86 +1417,6 @@ static int hts_rename(httrackp * opt, const char *a, const char *b) {
|
||||
return rename(a, b);
|
||||
}
|
||||
|
||||
/* Pathname of a file inside the mirror dir (rotating concat buffer). */
|
||||
static char *reconcile_path(httrackp *opt, const char *name) {
|
||||
return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), name);
|
||||
}
|
||||
|
||||
/* Interrupted-run heuristic: prefer the old generation when the new cache
|
||||
stalled below NEW_TINY while the old one grew past OLD_SOLID (historical
|
||||
arbitrary thresholds). */
|
||||
#define CACHE_RECONCILE_NEW_TINY 32768
|
||||
#define CACHE_RECONCILE_OLD_SOLID 65536
|
||||
|
||||
/* Replace the new-generation file by the old one, when the old one exists. */
|
||||
static void reconcile_promote(httrackp *opt, const char *oldname,
|
||||
const char *newname) {
|
||||
if (fexist(reconcile_path(opt, oldname))) {
|
||||
remove(reconcile_path(opt, newname));
|
||||
rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
|
||||
}
|
||||
}
|
||||
|
||||
void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
|
||||
switch (mode) {
|
||||
case CACHE_RECONCILE_PROMOTE:
|
||||
/* Previous run rotated new.* to old.* then died before writing: promote
|
||||
the old generation back, whichever format it uses. */
|
||||
if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if ((!fexist(reconcile_path(opt, "hts-cache/new.dat")) ||
|
||||
!fexist(reconcile_path(opt, "hts-cache/new.ndx"))) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
break;
|
||||
case CACHE_RECONCILE_INTERRUPTED:
|
||||
/* Aborted run: keep the larger generation when the new cache is
|
||||
suspiciously small next to the old one. The new file must exist: fsize()
|
||||
is -1 for a missing file, which would spuriously pass the "< TINY" test
|
||||
and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
|
||||
*/
|
||||
if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
|
||||
break;
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.dat")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.dat")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.dat")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.dat"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
break;
|
||||
case CACHE_RECONCILE_ROLLBACK:
|
||||
/* Nothing transferred: restore the previous generation and sidecars. */
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if (fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
reconcile_promote(opt, "hts-cache/old.lst", "hts-cache/new.lst");
|
||||
reconcile_promote(opt, "hts-cache/old.txt", "hts-cache/new.txt");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// renvoyer uniquement en tête, ou NULL si erreur
|
||||
// return NULL upon error, and set -1 to r.statuscode
|
||||
htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
|
||||
|
||||
@@ -66,11 +66,8 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, char *location);
|
||||
htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, char *location);
|
||||
/* Like cache_read, but also yields entries whose transfer broke; return_save
|
||||
(optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
|
||||
htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
|
||||
const char *adr, const char *fil,
|
||||
char *return_save);
|
||||
htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
|
||||
const char *adr, const char *fil);
|
||||
htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, char *location,
|
||||
char *return_save, int readonly);
|
||||
@@ -78,17 +75,6 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, htsblk * r);
|
||||
void cache_init(cache_back * cache, httrackp * opt);
|
||||
|
||||
/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
|
||||
typedef enum {
|
||||
CACHE_RECONCILE_PROMOTE, /* no new cache: promote the old generation */
|
||||
CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
|
||||
CACHE_RECONCILE_ROLLBACK /* nothing transferred: restore the old one */
|
||||
} hts_cache_reconcile_mode;
|
||||
|
||||
/* Reconcile the on-disk cache generations according to mode; a no-op when
|
||||
the involved files are absent. */
|
||||
void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
|
||||
|
||||
int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
|
||||
const char *str2, char *outbuff, int len);
|
||||
int cache_readdata(cache_back * cache, const char *str1, const char *str2,
|
||||
|
||||
@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htszlib.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
@@ -322,7 +321,6 @@ typedef struct {
|
||||
size_t budget; /**< bytes allowed through before writes start failing */
|
||||
int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
|
||||
int writes; /**< zwrite call count, to detect re-entry into the stream */
|
||||
int fail_once; /**< recover (unlimited budget) after the first failure */
|
||||
} writefail_inject;
|
||||
|
||||
/* zwrite that copies until the budget runs out, then fails with inj->fail_errno
|
||||
@@ -337,8 +335,6 @@ static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
|
||||
inj->budget -= (size_t) size;
|
||||
return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
|
||||
}
|
||||
if (inj->fail_once)
|
||||
inj->budget = (size_t) -1; /* the backend recovers after this failure */
|
||||
errno = inj->fail_errno;
|
||||
return 0; /* short write -> the minizip op returns an error */
|
||||
}
|
||||
@@ -377,50 +373,9 @@ static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
|
||||
freet(bodycopy);
|
||||
}
|
||||
|
||||
/* Store an entry claiming a >2GB body; the degrade path never reads data. */
|
||||
static void writefail_store_oversized(httrackp *opt, cache_back *cache,
|
||||
const char *fil, int is_write) {
|
||||
htsblk r;
|
||||
char locbuf[4];
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
r.statuscode = 200;
|
||||
r.size = (LLint) INT_MAX + 1;
|
||||
strcpybuff(r.msg, "OK");
|
||||
strcpybuff(r.contenttype, "application/octet-stream");
|
||||
locbuf[0] = '\0';
|
||||
r.location = locbuf;
|
||||
r.is_write = (short int) is_write;
|
||||
cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
|
||||
}
|
||||
|
||||
/* Read back `entryname`: extra field (cached headers) and body. Returns the
|
||||
body length, or -1 if the entry is absent or unreadable. */
|
||||
static int writefail_read_entry(const char *path, const char *entryname,
|
||||
char *extra, size_t extralen, char *body,
|
||||
size_t bodylen) {
|
||||
unzFile z = unzOpen(path);
|
||||
int n = -1;
|
||||
|
||||
if (z == NULL)
|
||||
return -1;
|
||||
if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
|
||||
unzOpenCurrentFile(z) == UNZ_OK) {
|
||||
const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
|
||||
|
||||
if (elen >= 0) {
|
||||
extra[elen] = '\0';
|
||||
n = unzReadCurrentFile(z, body, (unsigned) bodylen);
|
||||
}
|
||||
unzCloseCurrentFile(z);
|
||||
}
|
||||
unzClose(z);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
|
||||
stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
|
||||
*/
|
||||
/* #174/#219: a failing cache write used to crash via assertf(); it must instead
|
||||
stop the mirror (exit_xh = -1) without crashing. Assert that, plus the cache
|
||||
is flagged and a sibling write doesn't re-enter the broken stream. */
|
||||
int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
int fail = 0;
|
||||
char path[HTS_URLMAXSIZE];
|
||||
@@ -433,8 +388,9 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
gen_body(body, body_len, 1 /* incompressible */);
|
||||
fconcat(path, sizeof(path), dir, "/wfail.zip");
|
||||
|
||||
/* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
|
||||
drops entries until the streak caps out, then aborts. */
|
||||
/* phase 0: fail on the body write, fatal errno (ENOSPC, the disk-full
|
||||
branch). phase 1: fail on the open, non-fatal errno (EIO, dropped-share
|
||||
branch). Both must abort the mirror. */
|
||||
for (phase = 0; phase < 2; phase++) {
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
@@ -443,7 +399,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
inj.budget = (phase == 0) ? 4096 : 0;
|
||||
inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
@@ -457,25 +412,7 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
}
|
||||
|
||||
opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
|
||||
if (phase == 0) {
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
} else {
|
||||
/* the abort must land exactly on the 8th consecutive failure */
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 7; i++) {
|
||||
char fil[32];
|
||||
|
||||
snprintf(fil, sizeof(fil), "/b%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
}
|
||||
if (cache.zipWriteFailed) {
|
||||
fprintf(stderr, "cache-writefail: phase 1: aborted before the "
|
||||
"8th consecutive failure\n");
|
||||
fail++;
|
||||
}
|
||||
writefail_store(opt, &cache, "/b7.bin", body, 16);
|
||||
}
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
if (!cache.zipWriteFailed) {
|
||||
fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
|
||||
phase);
|
||||
@@ -506,136 +443,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
}
|
||||
}
|
||||
|
||||
/* failures with successes in between reset the streak: never aborts */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
int i;
|
||||
|
||||
inj.budget = (size_t) -1;
|
||||
inj.fail_errno = EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
for (i = 0; i < 10; i++) {
|
||||
char fil[32];
|
||||
|
||||
inj.budget = 0; /* this store fails */
|
||||
snprintf(fil, sizeof(fil), "/s%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
|
||||
snprintf(fil, sizeof(fil), "/ok%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
}
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: scattered: non-consecutive failures aborted "
|
||||
"the mirror (flagged=%d, exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
}
|
||||
|
||||
/* isolated failure: only that entry drops; a later sibling round-trips */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
char extra[8192];
|
||||
char rbody[64];
|
||||
int n;
|
||||
|
||||
inj.budget = 4096;
|
||||
inj.fail_errno = EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 1;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: skip: isolated failure aborted the mirror "
|
||||
"(flagged=%d, exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
writefail_store(opt, &cache, "/blob2.bin", body, 16);
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody));
|
||||
if (n != 16 || memcmp(rbody, body, 16) != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: skip: sibling entry lost after a skipped "
|
||||
"entry (%d)\n",
|
||||
n);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
|
||||
/* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
char extra[8192];
|
||||
char rbody[64];
|
||||
int n;
|
||||
|
||||
inj.budget = (size_t) -1; /* no injected failure */
|
||||
inj.fail_errno = 0;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
|
||||
writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: mirror aborted (flagged=%d, "
|
||||
"exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody)) >= 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: in-memory entry was stored\n");
|
||||
fail++;
|
||||
}
|
||||
n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody));
|
||||
if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: on-disk entry not stored "
|
||||
"headers-only (%d)\n",
|
||||
n);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
|
||||
freet(body);
|
||||
return fail;
|
||||
}
|
||||
@@ -909,478 +716,3 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* --- hts_cache_reconcile() policies -------------------------------------- */
|
||||
|
||||
/* All reconcile inputs/outputs, wiped between cases. */
|
||||
static const char *const reconcile_files[] = {
|
||||
"hts-cache/new.zip", "hts-cache/old.zip", "hts-cache/new.dat",
|
||||
"hts-cache/old.dat", "hts-cache/new.ndx", "hts-cache/old.ndx",
|
||||
"hts-cache/new.lst", "hts-cache/old.lst", "hts-cache/new.txt",
|
||||
"hts-cache/old.txt", "hts-in_progress.lock"};
|
||||
|
||||
static char *reconcile_st_path(httrackp *opt, const char *name) {
|
||||
return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), name);
|
||||
}
|
||||
|
||||
static void reconcile_wipe(httrackp *opt) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
|
||||
remove(reconcile_st_path(opt, reconcile_files[i]));
|
||||
}
|
||||
|
||||
/* Create a filler file of exactly `size` bytes. */
|
||||
static void reconcile_put(httrackp *opt, const char *name, size_t size) {
|
||||
FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
|
||||
static const char filler[1024] = {'x'};
|
||||
|
||||
assertf(fp != NULL);
|
||||
while (size > 0) {
|
||||
const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
|
||||
|
||||
assertf(fwrite(filler, 1, n, fp) == n);
|
||||
size -= n;
|
||||
}
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
|
||||
static int reconcile_expect(httrackp *opt, const char *name, off_t size,
|
||||
const char *what) {
|
||||
const off_t got = fsize(reconcile_st_path(opt, name));
|
||||
|
||||
if (got != size) {
|
||||
fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
|
||||
name, (int) got, (int) size);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cache_reconcile_selftest(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
|
||||
/* around the interrupted-run thresholds (new < 32768, old > 65536) */
|
||||
static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
|
||||
|
||||
golden_setup(opt, dir);
|
||||
#ifdef _WIN32
|
||||
mkdir(reconcile_st_path(opt, "hts-cache"));
|
||||
#else
|
||||
mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
|
||||
/* PROMOTE: a zip old generation replaces a missing new one */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
|
||||
|
||||
/* PROMOTE: an existing new.zip is left alone */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
|
||||
|
||||
/* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
|
||||
zip cache existed) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
|
||||
|
||||
/* PROMOTE: a half-written legacy new pair is replaced by the old pair */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
|
||||
|
||||
/* INTERRUPTED: no lock file, no action */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
|
||||
|
||||
/* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
|
||||
spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
|
||||
|
||||
/* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
|
||||
zip caches: the arm was gated on a legacy new.dat) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
|
||||
|
||||
/* INTERRUPTED: old below the confidence threshold, keep new */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
|
||||
|
||||
/* INTERRUPTED: new big enough to trust, keep it */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", MID);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
|
||||
|
||||
/* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/new.ndx", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
|
||||
|
||||
/* ROLLBACK: the old zip generation is restored (a zip cache used to lose
|
||||
its only good generation here) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
|
||||
|
||||
/* ROLLBACK: sidecars are restored regardless of format */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.lst", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.lst", MID);
|
||||
reconcile_put(opt, "hts-cache/old.txt", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
|
||||
|
||||
/* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/new.ndx", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", MID);
|
||||
reconcile_put(opt, "hts-cache/old.lst", MID);
|
||||
reconcile_put(opt, "hts-cache/old.txt", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
|
||||
|
||||
/* ROLLBACK: nothing to restore, the new generation stays */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
|
||||
|
||||
reconcile_wipe(opt);
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* --- read-side corruption injection --------------------------------------- */
|
||||
|
||||
/* canary read back intact after each corruption; victim gets the byte surgery
|
||||
*/
|
||||
#define CORRUPT_ADR "corrupt.example.com"
|
||||
static char corrupt_body_a[33 + 1];
|
||||
static char corrupt_body_b[44 + 1];
|
||||
|
||||
/* Write a fresh two-entry cache: /canary.html then /victim.html. */
|
||||
static void corrupt_build(httrackp *opt) {
|
||||
cache_back cache;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
|
||||
strlen(corrupt_body_b));
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
|
||||
is later overwritten with a forged oversized X-Size (same byte length). */
|
||||
static void corrupt_build_etag(httrackp *opt) {
|
||||
cache_back cache;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
|
||||
corrupt_body_b, strlen(corrupt_body_b));
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
|
||||
body on disk): the shape every non-html file is stored with. */
|
||||
static void corrupt_build_disk(httrackp *opt) {
|
||||
cache_back cache;
|
||||
htsblk w;
|
||||
char locw[4];
|
||||
char BIGSTK save[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
|
||||
char *path;
|
||||
FILE *fp;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
|
||||
CORRUPT_ADR "/victim.bin");
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
hts_init_htsblk(&w);
|
||||
w.statuscode = 200;
|
||||
w.size = (LLint) sizeof(corrupt_body_b) - 1;
|
||||
strcpybuff(w.msg, "OK");
|
||||
strcpybuff(w.contenttype, "application/octet-stream");
|
||||
strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
|
||||
locw[0] = '\0';
|
||||
w.location = locw;
|
||||
w.is_write = 0;
|
||||
cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
|
||||
0 /* all_in_cache */, NULL);
|
||||
selftest_close(&cache);
|
||||
/* the reader only checks this file exists; it never reads it here */
|
||||
path = fconv(catbuff, sizeof(catbuff), save);
|
||||
(void) structcheck(path);
|
||||
fp = FOPEN(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
|
||||
static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
|
||||
const char *rep, size_t nth, size_t total) {
|
||||
LLint fsz = 0;
|
||||
char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
|
||||
const size_t n = (size_t) fsz;
|
||||
size_t k, hits = 0, at = 0;
|
||||
FILE *fp;
|
||||
|
||||
assertf(data != NULL);
|
||||
for (k = 0; k + patlen <= n; k++) {
|
||||
if (memcmp(data + k, pat, patlen) == 0) {
|
||||
hits++;
|
||||
if (hits == nth)
|
||||
at = k;
|
||||
}
|
||||
}
|
||||
assertf(hits == total);
|
||||
memcpy(data + at, rep, patlen);
|
||||
fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, n, fp) == n);
|
||||
fclose(fp);
|
||||
freet(data);
|
||||
}
|
||||
|
||||
/* Garbage the first bytes of the victim's deflated data (2nd local header). */
|
||||
static void corrupt_victim_body(httrackp *opt) {
|
||||
LLint fsz = 0;
|
||||
char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
|
||||
const size_t n = (size_t) fsz;
|
||||
size_t k, hits = 0, off = 0;
|
||||
FILE *fp;
|
||||
|
||||
assertf(data != NULL);
|
||||
for (k = 0; k + 4 <= n; k++) {
|
||||
if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
|
||||
const size_t namelen =
|
||||
(unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
|
||||
const size_t extralen =
|
||||
(unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
|
||||
|
||||
off = k + 30 + namelen + extralen;
|
||||
}
|
||||
}
|
||||
assertf(hits == 2);
|
||||
assertf(off != 0 && off + 4 <= n);
|
||||
memset(data + off, 0xFF, 4);
|
||||
fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, n, fp) == n);
|
||||
fclose(fp);
|
||||
freet(data);
|
||||
}
|
||||
|
||||
/* Read the corrupt /victim.html and, in the SAME read session, the intact
|
||||
/canary.html: the victim must be rejected (wantmsg pins which path) and the
|
||||
canary must still decode byte-exact, proving one bad entry never taints a
|
||||
sibling read. */
|
||||
static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
|
||||
const char *wantmsg, const char *what) {
|
||||
cache_back cache;
|
||||
htsblk v, c;
|
||||
char BIGSTK lv[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK lc[HTS_URLMAXSIZE * 2];
|
||||
int fail = 0;
|
||||
|
||||
selftest_open_for_read(&cache, opt);
|
||||
lv[0] = lc[0] = '\0';
|
||||
v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
|
||||
if (v.statuscode != STATUSCODE_INVALID) {
|
||||
fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
|
||||
selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
|
||||
fail++;
|
||||
}
|
||||
if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
|
||||
fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
|
||||
selftest_tag, what, v.msg, wantmsg);
|
||||
fail++;
|
||||
}
|
||||
c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
|
||||
if (c.statuscode != 200 || c.adr == NULL ||
|
||||
c.size != (LLint) strlen(corrupt_body_a) ||
|
||||
memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
|
||||
fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
|
||||
c.statuscode);
|
||||
fail++;
|
||||
}
|
||||
if (v.adr != NULL)
|
||||
freet(v.adr);
|
||||
if (c.adr != NULL)
|
||||
freet(c.adr);
|
||||
selftest_close(&cache);
|
||||
return fail;
|
||||
}
|
||||
|
||||
static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
|
||||
const char *what) {
|
||||
return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
|
||||
}
|
||||
|
||||
/* Headers-only probe of the disk victim: must parse OK with the size kept. */
|
||||
static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
|
||||
const char *what) {
|
||||
cache_back cache;
|
||||
htsblk v;
|
||||
char BIGSTK lv[HTS_URLMAXSIZE * 2];
|
||||
int fail = 0;
|
||||
|
||||
selftest_open_for_read(&cache, opt);
|
||||
lv[0] = '\0';
|
||||
v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
|
||||
if (v.statuscode != 200 || v.size != wantsize) {
|
||||
fprintf(stderr,
|
||||
"%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
|
||||
selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
|
||||
fail++;
|
||||
}
|
||||
if (v.adr != NULL)
|
||||
freet(v.adr);
|
||||
selftest_close(&cache);
|
||||
return fail;
|
||||
}
|
||||
|
||||
/* One zip corruption case: build, patch, then check victim+canary in-session.
|
||||
*/
|
||||
static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
|
||||
size_t nth, size_t total, const char *wantmsg,
|
||||
const char *what) {
|
||||
corrupt_build(opt);
|
||||
corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
|
||||
return corrupt_expect_victim(opt, wantmsg, what);
|
||||
}
|
||||
|
||||
int cache_corruption_selftest(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
|
||||
selftest_tag = "cache-corrupt";
|
||||
golden_setup(opt, dir);
|
||||
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
|
||||
"Cache Read Error : Read Data", "oversized X-Size");
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
|
||||
"Cache Read Error : Bad Size", "negative X-Size");
|
||||
/* both entries carry the line; the victim's is the second */
|
||||
failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
|
||||
"Previous cache file not found (empty filename)",
|
||||
"blanked X-In-Cache");
|
||||
/* smashed local file header: the entry is dropped at index load */
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
|
||||
"File Cache Entry Not Found", "smashed local header");
|
||||
|
||||
corrupt_build(opt);
|
||||
corrupt_victim_body(opt);
|
||||
failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
|
||||
"garbled deflate stream");
|
||||
|
||||
/* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
|
||||
truncates negative in the (int) cast the malloc uses: a wraparound alloc.
|
||||
cache_add asserts size fits an int, so such a value only reaches the reader
|
||||
from a corrupt/foreign cache; inject it by overwriting the victim's long
|
||||
Etag line with a same-length forged X-Size line (the parser keeps the last
|
||||
X-Size it sees), keeping the zip byte-length and offsets intact. */
|
||||
corrupt_build_etag(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483648AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
|
||||
"X-Size above INT_MAX");
|
||||
|
||||
/* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
|
||||
is how every >2GB non-html file is stored. It must survive a header probe
|
||||
(or every update re-fetches the file); an in-memory read still rejects. */
|
||||
corrupt_build_disk(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483648AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
|
||||
"headers-only X-Size above INT_MAX");
|
||||
failures += corrupt_expect_victim_fil(opt, "/victim.bin",
|
||||
"Cache Read Error : Bad Size",
|
||||
"in-memory X-Size above INT_MAX");
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
@@ -52,19 +52,10 @@ int cache_selftests(httrackp *opt, const char *dir);
|
||||
committed file, never by the test). Returns the failed-check count. */
|
||||
int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
|
||||
|
||||
/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
|
||||
drop just the entry otherwise. Returns the failed-check count. */
|
||||
/* #174/#219: assert a failing cache write aborts the mirror cleanly instead of
|
||||
crashing. Returns the failed-check count. */
|
||||
int cache_write_failure_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
/* Exercise the hts_cache_reconcile() generation policies on file fixtures
|
||||
under <dir>. Returns the failed-check count. */
|
||||
int cache_reconcile_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
|
||||
under <dir> and assert every case degrades to STATUSCODE_INVALID without
|
||||
tainting a sibling entry. */
|
||||
int cache_corruption_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -175,9 +175,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
|
||||
//
|
||||
socinput(soc, line, 1000);
|
||||
if (strnotempty(line)) {
|
||||
/* widths bound the caller buffers: method[32], url[HTS_URLMAXSIZE*2],
|
||||
protocol[256] */
|
||||
if (sscanf(line, "%31s %2047s %255s", method, url, protocol) == 3) {
|
||||
if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
|
||||
lien_adrfil af;
|
||||
|
||||
// méthode en majuscule
|
||||
|
||||
204
src/htscore.c
204
src/htscore.c
@@ -406,106 +406,29 @@ void hts_invalidate_link(httrackp * opt, int lpos) {
|
||||
opt->liens[lpos]->pass2 = -1;
|
||||
}
|
||||
|
||||
// Write the makeindex footer (refresh meta when makeindex_links==1), close
|
||||
// the file, then run usercommand.
|
||||
void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
|
||||
FILE **makeindex_fp, int makeindex_links,
|
||||
const char *makeindex_firstlink,
|
||||
const char *template_footer, const char *adr,
|
||||
const char *fil) {
|
||||
if (!*makeindex_done) {
|
||||
if (*makeindex_fp) {
|
||||
char BIGSTK tempo[1024];
|
||||
if (makeindex_links == 1) {
|
||||
char BIGSTK link_escaped[HTS_URLMAXSIZE * 2];
|
||||
escape_uri_utf(makeindex_firstlink, link_escaped, sizeof(link_escaped));
|
||||
snprintf(tempo, sizeof(tempo),
|
||||
"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">" CRLF,
|
||||
link_escaped);
|
||||
} else
|
||||
tempo[0] = '\0';
|
||||
hts_template_format(*makeindex_fp, template_footer,
|
||||
"<!-- Mirror and index made by HTTrack Website "
|
||||
"Copier/" HTTRACK_VERSION " " HTTRACK_AFF_AUTHORS
|
||||
" -->",
|
||||
tempo, /* EOF */ NULL);
|
||||
fflush(*makeindex_fp);
|
||||
fclose(*makeindex_fp);
|
||||
*makeindex_fp = NULL;
|
||||
usercommand(opt, 0, NULL,
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html_utf8), "index.html"),
|
||||
adr, fil);
|
||||
}
|
||||
}
|
||||
*makeindex_done = 1;
|
||||
}
|
||||
|
||||
/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
|
||||
* on-disk MD5 is unchanged. */
|
||||
void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
|
||||
FILE **fp, const char *ht_buff, size_t ht_len,
|
||||
const char *adr, const char *fil, const char *save) {
|
||||
char digest[32 + 2];
|
||||
off_t fsize_old =
|
||||
fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
|
||||
int ok = 0;
|
||||
|
||||
digest[0] = '\0';
|
||||
domd5mem(ht_buff, ht_len, digest, 1);
|
||||
if (fsize_old == (off_t) ht_len) {
|
||||
int mlen = 0;
|
||||
char *mbuff;
|
||||
|
||||
cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
|
||||
if (mlen)
|
||||
mbuff[mlen] = '\0';
|
||||
if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
|
||||
ok = 1;
|
||||
hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
|
||||
}
|
||||
freet(mbuff);
|
||||
}
|
||||
if (!ok) {
|
||||
file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
|
||||
*fp = filecreate(&opt->state.strc, save);
|
||||
if (*fp) {
|
||||
if (ht_len > 0 && fwrite(ht_buff, 1, ht_len, *fp) != ht_len) {
|
||||
int fcheck = check_fatal_io_errno();
|
||||
|
||||
if (fcheck)
|
||||
opt->state.exit_xh = -1;
|
||||
if (opt->log) {
|
||||
hts_log_print(opt, LOG_ERROR | LOG_ERRNO,
|
||||
"Unable to write HTML file %s", save);
|
||||
if (fcheck)
|
||||
hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
|
||||
}
|
||||
}
|
||||
fclose(*fp);
|
||||
*fp = NULL;
|
||||
if (strnotempty(r->lastmodified))
|
||||
set_filetime_rfc822(save, r->lastmodified);
|
||||
} else {
|
||||
int fcheck = check_fatal_io_errno();
|
||||
|
||||
if (fcheck) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: disk full or filesystem problems");
|
||||
opt->state.exit_xh = -1;
|
||||
}
|
||||
hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to save file %s", save);
|
||||
if (fcheck)
|
||||
hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
|
||||
}
|
||||
} else {
|
||||
file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
|
||||
filenote(&opt->state.strc, save, NULL);
|
||||
}
|
||||
if (cache->ndx)
|
||||
cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
|
||||
(int) strlen(digest));
|
||||
}
|
||||
#define HT_INDEX_END do { \
|
||||
if (!makeindex_done) { \
|
||||
if (makeindex_fp) { \
|
||||
char BIGSTK tempo[1024]; \
|
||||
if (makeindex_links == 1) { \
|
||||
char BIGSTK link_escaped[HTS_URLMAXSIZE*2]; \
|
||||
escape_uri_utf(makeindex_firstlink, link_escaped, sizeof(link_escaped)); \
|
||||
snprintf(tempo,sizeof(tempo),"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">"CRLF, link_escaped); \
|
||||
} else \
|
||||
tempo[0]='\0'; \
|
||||
hts_template_format(makeindex_fp,template_footer, \
|
||||
"<!-- Mirror and index made by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->", \
|
||||
tempo, /* EOF */ NULL \
|
||||
); \
|
||||
fflush(makeindex_fp); \
|
||||
fclose(makeindex_fp); /* à ne pas oublier sinon on passe une nuit blanche */ \
|
||||
makeindex_fp=NULL; \
|
||||
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_html_utf8),"index.html"),"",""); \
|
||||
} \
|
||||
} \
|
||||
makeindex_done=1; /* ok c'est fait */ \
|
||||
} while(0)
|
||||
|
||||
/* does it look like XML ? (SVG et al.) */
|
||||
static int look_like_xml(const char *s) {
|
||||
@@ -2121,8 +2044,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
/*
|
||||
Ensure the index is being closed
|
||||
*/
|
||||
hts_finish_makeindex(opt, &makeindex_done, &makeindex_fp, makeindex_links,
|
||||
makeindex_firstlink, template_footer, "", "");
|
||||
HT_INDEX_END;
|
||||
|
||||
/*
|
||||
updating-a-remotely-deteted-website hack
|
||||
@@ -2137,7 +2059,47 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
hts_log_print(opt, LOG_NOTICE,
|
||||
"No data seems to have been transferred during this session! : restoring previous one!");
|
||||
XH_uninit;
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/old.dat")))
|
||||
&&
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))) {
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"));
|
||||
}
|
||||
opt->state.exit_xh = 2; /* interrupted (no connection detected) */
|
||||
return 1;
|
||||
}
|
||||
@@ -2852,9 +2814,6 @@ int check_fatal_io_errno(void) {
|
||||
#endif
|
||||
#ifdef EROFS
|
||||
case EROFS: /* Read-only file system */
|
||||
#endif
|
||||
#ifdef EDQUOT
|
||||
case EDQUOT: /* Disk quota exceeded */
|
||||
#endif
|
||||
return 1;
|
||||
break;
|
||||
@@ -3334,41 +3293,6 @@ int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt) {
|
||||
return n;
|
||||
}
|
||||
|
||||
/* One engine-loop tick: refresh the transfer stats and run the loop callback
|
||||
for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
|
||||
hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr) {
|
||||
engine_stats();
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
return RUN_CALLBACK7(
|
||||
opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)
|
||||
? HTS_TRUE
|
||||
: HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Single implementation of the historical WAIT_FOR_AVAILABLE_SOCKET macros. */
|
||||
hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
|
||||
cache_back *cache, int ptr) {
|
||||
const int prev = opt->state._hts_in_html_parsing;
|
||||
|
||||
while (back_pluggable_sockets_strict(sback, opt) <= 0) {
|
||||
opt->state._hts_in_html_parsing = 6;
|
||||
back_wait(sback, opt, cache, 0);
|
||||
/* time limit (-E) exceeded: stop waiting for a socket (#481) */
|
||||
if (!back_checkmirror(opt))
|
||||
break;
|
||||
if (!hts_loop_tick(sback, opt, -1, ptr))
|
||||
return HTS_FALSE;
|
||||
}
|
||||
opt->state._hts_in_html_parsing = prev;
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
int back_pluggable_sockets(struct_back * sback, httrackp * opt) {
|
||||
int n;
|
||||
|
||||
|
||||
@@ -216,7 +216,6 @@ struct cache_back {
|
||||
int zipEntriesCapa;
|
||||
hts_boolean
|
||||
zipWriteFailed; /**< a cache write failed; stop touching the stream */
|
||||
int zipWriteFailures; /**< consecutive entry write failures; reset on store */
|
||||
};
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_hash_struct
|
||||
@@ -363,20 +362,6 @@ void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
|
||||
|
||||
void usercommand_exe(const char *cmd, const char *file);
|
||||
|
||||
// Finish the makeindex index.html (footer + refresh meta), run usercommand.
|
||||
// Updates *makeindex_done/*makeindex_fp in place; adr/fil are the mode strings.
|
||||
void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
|
||||
FILE **makeindex_fp, int makeindex_links,
|
||||
const char *makeindex_firstlink,
|
||||
const char *template_footer, const char *adr,
|
||||
const char *fil);
|
||||
|
||||
// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
|
||||
// closed+NULLed on write. Precondition: ht_len>0.
|
||||
void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
|
||||
FILE **fp, const char *ht_buff, size_t ht_len,
|
||||
const char *adr, const char *fil, const char *save);
|
||||
|
||||
int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
|
||||
|
||||
int fspc(httrackp * opt, FILE * fp, const char *type);
|
||||
@@ -433,15 +418,6 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
|
||||
|
||||
int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
|
||||
|
||||
/* One engine-loop tick: refresh the transfer stats and run the loop callback
|
||||
for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
|
||||
hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
|
||||
|
||||
/* Wait until a test socket can be plugged, pumping transfers, stats and the
|
||||
loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
|
||||
hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
|
||||
cache_back *cache, int ptr);
|
||||
|
||||
/* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
|
||||
timestamp seed so it is stable within one gap and rerolls per launch. */
|
||||
int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);
|
||||
@@ -494,8 +470,4 @@ void voidf(void);
|
||||
/* HTML marker comment marking where the top index is spliced. */
|
||||
#define HTS_TOPINDEX "TOP_INDEX_HTTRACK"
|
||||
|
||||
/* Worst-case byte expansion HT_ADD_HTMLESCAPED* must reserve per escaper. */
|
||||
#define HTS_HTMLESCAPE_MAXEXP 5 /* escape_for_html_print: '&'->"&" */
|
||||
#define HTS_HTMLESCAPE_FULL_MAXEXP 6 /* _full: high byte->"&#xHH;" */
|
||||
|
||||
#endif
|
||||
|
||||
@@ -544,11 +544,69 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
}
|
||||
|
||||
// No new cache but an old one? promote it
|
||||
// Existence d'un cache - pas de new mais un old.. renommer
|
||||
#if DEBUG_STEPS
|
||||
printf("Checking cache\n");
|
||||
#endif
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
if (!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip"))) {
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
}
|
||||
} else
|
||||
if ((!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.dat")))
|
||||
||
|
||||
(!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))) {
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.dat")))
|
||||
&&
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))) {
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
//remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
//rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
|
||||
}
|
||||
}
|
||||
|
||||
/* Interrupted mirror detected */
|
||||
if (!opt->quiet) {
|
||||
@@ -2496,8 +2554,109 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
printf("Cache & log settings\n");
|
||||
#endif
|
||||
|
||||
// If both cache generations exist, keep the most complete one
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
// on utilise le cache..
|
||||
// en cas de présence des deux versions, garder la version la plus avancée,
|
||||
// cad la version contenant le plus de fichiers
|
||||
if (opt->cache) {
|
||||
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) { // problemes..
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")) < 32768) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) > 65536) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->
|
||||
path_log),
|
||||
"hts-cache/new.zip")))
|
||||
{
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))
|
||||
&&
|
||||
fexist(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"))
|
||||
&&
|
||||
fexist(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"))) {
|
||||
// switcher si new<32Ko et old>65Ko (tailles arbitraires) ?
|
||||
// ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre
|
||||
// alors l'ancien cache
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat")) < 32768) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")) > 65536) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->
|
||||
path_log),
|
||||
"hts-cache/new.dat")))
|
||||
{
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
//} else { // ne rien faire
|
||||
// remove("hts-cache/old.dat");
|
||||
// remove("hts-cache/old.ndx");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Débuggage des en têtes
|
||||
if (_DEBUG_HEAD) {
|
||||
ioinfo =
|
||||
|
||||
@@ -69,15 +69,11 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
|
||||
typedef struct t_hts_callbackarg t_hts_callbackarg;
|
||||
#endif
|
||||
|
||||
/* Marks a symbol an external wrapper module exports back to the engine.
|
||||
Must override -fvisibility=hidden on ELF, or dlopen()ed plugins (htsjava)
|
||||
hide their own hts_plug()/hts_unplug() entry points. */
|
||||
/* Marks a symbol an external wrapper module exports back to the engine
|
||||
(dllexport on Windows, nothing elsewhere). */
|
||||
#ifndef EXTERNAL_FUNCTION
|
||||
#ifdef _WIN32
|
||||
#define EXTERNAL_FUNCTION __declspec(dllexport)
|
||||
#elif ((defined(__GNUC__) && (__GNUC__ >= 4)) || \
|
||||
(defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
|
||||
#define EXTERNAL_FUNCTION __attribute__((visibility("default")))
|
||||
#else
|
||||
#define EXTERNAL_FUNCTION
|
||||
#endif
|
||||
|
||||
@@ -190,9 +190,9 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* reserve one byte for the trailing NUL written after the loop */
|
||||
if (j + 1 >= max) {
|
||||
|
||||
/* copy */
|
||||
if (j + 1 > max) {
|
||||
/* overflow */
|
||||
return -1;
|
||||
}
|
||||
@@ -300,11 +300,6 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
|
||||
|
||||
/* Was the character read successfully ? */
|
||||
if (nRead == utfBufferSize) {
|
||||
/* the 'continue' below skips the NUL-reserve guard: re-check */
|
||||
if (utfBufferJ + utfBufferSize >= max) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Rollback write position to sequence start write position */
|
||||
j = utfBufferJ;
|
||||
|
||||
@@ -319,8 +314,8 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
|
||||
}
|
||||
}
|
||||
|
||||
/* reserve one byte for the trailing NUL written after the loop */
|
||||
if (j + 1 >= max) {
|
||||
/* Check for overflow */
|
||||
if (j + 1 > max) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
50
src/htsftp.c
50
src/htsftp.c
@@ -128,33 +128,6 @@ void launch_ftp(FTPDownloadStruct * params) {
|
||||
return 0; \
|
||||
}
|
||||
|
||||
/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
|
||||
void ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size) {
|
||||
size_t n = 0;
|
||||
|
||||
assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
|
||||
|
||||
while (src[n] != '\0' && src[n] != ':') {
|
||||
if (n < user_size - 1)
|
||||
user[n] = src[n];
|
||||
n++;
|
||||
}
|
||||
user[n < user_size ? n : user_size - 1] = '\0';
|
||||
pass[0] = '\0';
|
||||
if (src[n] == ':') { // password follows the colon
|
||||
const size_t base = n + 1;
|
||||
size_t k = 0;
|
||||
|
||||
while (&src[base + k + 1] < end && src[base + k] != '\0') {
|
||||
if (k < pass_size - 1)
|
||||
pass[k] = src[base + k];
|
||||
k++;
|
||||
}
|
||||
pass[k < pass_size ? k : pass_size - 1] = '\0';
|
||||
}
|
||||
}
|
||||
|
||||
// la véritable fonction une fois lancées les routines thread/fork
|
||||
int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
lien_back *back = pStruct->pBack;
|
||||
@@ -200,7 +173,24 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
while(*real_adr == '/')
|
||||
real_adr++; // sauter /
|
||||
if ((adr = jump_identification(real_adr)) != real_adr) { // user
|
||||
ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
|
||||
int i = -1;
|
||||
|
||||
pass[0] = '\0';
|
||||
do {
|
||||
i++;
|
||||
user[i] = real_adr[i];
|
||||
} while((real_adr[i] != ':') && (real_adr[i]));
|
||||
user[i] = '\0';
|
||||
if (real_adr[i] == ':') { // pass
|
||||
int j = -1;
|
||||
|
||||
i++; // oui on saute aussi le :
|
||||
do {
|
||||
j++;
|
||||
pass[j] = real_adr[i + j];
|
||||
} while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j]));
|
||||
pass[j] = '\0';
|
||||
}
|
||||
}
|
||||
// Calculer RETR <nom>
|
||||
{
|
||||
@@ -994,8 +984,8 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
|
||||
//case 0: break; // pas encore --> erreur (on attend)!
|
||||
case 1:
|
||||
HTS_STAT.HTS_TOTAL_RECV += 1; // compter flux entrant
|
||||
if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
|
||||
data[i++] = b; // truncate hostile over-long reply lines
|
||||
if ((b != 10) && (b != 13))
|
||||
data[i++] = b;
|
||||
break;
|
||||
default:
|
||||
if (ptrline)
|
||||
|
||||
@@ -70,11 +70,6 @@ int back_launch_ftp(FTPDownloadStruct * params);
|
||||
int run_launch_ftp(FTPDownloadStruct * params);
|
||||
int send_line(T_SOC soc, const char *data);
|
||||
int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
|
||||
/* Split a "user[:pass]@" prefix (end = jump_identification result) into
|
||||
bounded, NUL-terminated user/pass buffers, truncating to fit.
|
||||
Both sizes must be nonzero. */
|
||||
void ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size);
|
||||
T_SOC get_datasocket(char *to_send, size_t to_send_size);
|
||||
int stop_ftp(lien_back * back);
|
||||
char *linejmp(char *line);
|
||||
|
||||
@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
|
||||
configure.ac, decoupled from these). VERSION is the display form, VERSIONID
|
||||
the dotted numeric form, AFF_VERSION the short form shown in footers,
|
||||
LIB_VERSION the data/cache format generation. */
|
||||
#define HTTRACK_VERSION "3.49-11"
|
||||
#define HTTRACK_VERSIONID "3.49.11"
|
||||
#define HTTRACK_VERSION "3.49-10"
|
||||
#define HTTRACK_VERSIONID "3.49.10"
|
||||
#define HTTRACK_AFF_VERSION "3.x"
|
||||
#define HTTRACK_LIB_VERSION "2.0"
|
||||
|
||||
|
||||
@@ -63,9 +63,6 @@ Please visit our Website: http://www.httrack.com
|
||||
/* This file */
|
||||
#include "htsjava.h"
|
||||
|
||||
/* calloct/freet wrappers */
|
||||
#include "htssafe.h"
|
||||
|
||||
static int reverse_endian(void) {
|
||||
int endian = 1;
|
||||
|
||||
@@ -207,16 +204,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* A constant-pool entry is >= 1 byte on disk; reject a count exceeding
|
||||
the file size (hostile .class ~68 MB alloc DoS). */
|
||||
if (!hts_count_fits(header.count, (LLint) fsize(file))) {
|
||||
fclose(fpout);
|
||||
sprintf(str->err_msg,
|
||||
"Invalid constant pool count %u (file len " LLintP ")",
|
||||
(unsigned) header.count, (LLint) fsize(file));
|
||||
return 0;
|
||||
}
|
||||
tab = (RESP_STRUCT *) calloct(header.count, sizeof(RESP_STRUCT));
|
||||
tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
|
||||
if (!tab) {
|
||||
sprintf(str->err_msg, "Unable to alloc %d bytes",
|
||||
(int) sizeof(RESP_STRUCT));
|
||||
@@ -242,7 +230,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
} else { // ++ une erreur est survenue!
|
||||
if (strnotempty(str->err_msg) == 0)
|
||||
strcpy(str->err_msg, "Internal readtable error");
|
||||
freet(tab);
|
||||
free(tab);
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
@@ -300,7 +288,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
#if JAVADEBUG
|
||||
printf("end\n");
|
||||
#endif
|
||||
freet(tab);
|
||||
free(tab);
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
|
||||
@@ -33,19 +33,15 @@ Please visit our Website: http://www.httrack.com
|
||||
#ifndef HTSJAVA_DEFH
|
||||
#define HTSJAVA_DEFH
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
|
||||
#define HTS_DEF_FWSTRUCT_JAVA_HEADER
|
||||
typedef struct JAVA_HEADER JAVA_HEADER;
|
||||
#endif
|
||||
/* 10-byte on-disk .class header image, fread() directly: fields need exact
|
||||
widths (LP64's 8-byte 'unsigned long' magic never matched 0xCAFEBABE). */
|
||||
struct JAVA_HEADER {
|
||||
uint32_t magic;
|
||||
uint16_t minor;
|
||||
uint16_t major;
|
||||
uint16_t count;
|
||||
unsigned long int magic;
|
||||
unsigned short int minor;
|
||||
unsigned short int major;
|
||||
unsigned short int count;
|
||||
};
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT
|
||||
|
||||
53
src/htslib.c
53
src/htslib.c
@@ -1149,8 +1149,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
char BIGSTK protocol[256], url[HTS_URLMAXSIZE * 2], method[256];
|
||||
|
||||
linput(fp, line, 1000);
|
||||
/* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
|
||||
if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
|
||||
if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
|
||||
size_t ret;
|
||||
// selon que l'on a ou pas un proxy
|
||||
if (retour->req.proxy.active) {
|
||||
@@ -4132,32 +4131,24 @@ DECLARE_APPEND_ESCAPE_VERSION(escape_uri)
|
||||
|
||||
#undef DECLARE_APPEND_ESCAPE_VERSION
|
||||
|
||||
// In-place escaping: copy dest aside, then escape that copy back into dest.
|
||||
typedef size_t (*escape_fn_t)(const char *src, char *dest, size_t size);
|
||||
|
||||
static size_t inplace_escape(char *const dest, const size_t size,
|
||||
escape_fn_t escape) {
|
||||
char buffer[256];
|
||||
const size_t len = strnlen(dest, size);
|
||||
const int in_buffer = len + 1 < sizeof(buffer);
|
||||
char *src = in_buffer ? buffer : malloct(len + 1);
|
||||
size_t ret;
|
||||
assertf(src != NULL);
|
||||
assertf(len < size);
|
||||
memcpy(src, dest, len + 1);
|
||||
ret = escape(src, dest, size);
|
||||
if (!in_buffer) {
|
||||
freet(src);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Thin exported wrappers binding inplace_escape() to each escaper (ABI).
|
||||
// Same as above, but in-place
|
||||
#undef DECLARE_INPLACE_ESCAPE_VERSION
|
||||
#define DECLARE_INPLACE_ESCAPE_VERSION(NAME) \
|
||||
HTSEXT_API size_t inplace_##NAME(char *const dest, const size_t size) { \
|
||||
return inplace_escape(dest, size, NAME); \
|
||||
}
|
||||
#define DECLARE_INPLACE_ESCAPE_VERSION(NAME) \
|
||||
HTSEXT_API size_t inplace_ ##NAME(char *const dest, const size_t size) { \
|
||||
char buffer[256]; \
|
||||
const size_t len = strnlen(dest, size); \
|
||||
const int in_buffer = len + 1 < sizeof(buffer); \
|
||||
char *src = in_buffer ? buffer : malloct(len + 1); \
|
||||
size_t ret; \
|
||||
assertf(src != NULL); \
|
||||
assertf(len < size); \
|
||||
memcpy(src, dest, len + 1); \
|
||||
ret = NAME(src, dest, size); \
|
||||
if (!in_buffer) { \
|
||||
freet(src); \
|
||||
} \
|
||||
return ret; \
|
||||
}
|
||||
|
||||
DECLARE_INPLACE_ESCAPE_VERSION(escape_in_url)
|
||||
DECLARE_INPLACE_ESCAPE_VERSION(escape_spc_url)
|
||||
@@ -6023,11 +6014,9 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
"htsswf", "htsjava", "httrack-plugin", NULL
|
||||
};
|
||||
#else
|
||||
#ifndef HTS_LIBHTSJAVA_NAME
|
||||
#define HTS_LIBHTSJAVA_NAME "libhtsjava.so" /* non-autoconf fallback */
|
||||
#endif
|
||||
static const char *defaultModules[] = {"libhtsswf.so.1", HTS_LIBHTSJAVA_NAME,
|
||||
"httrack-plugin", NULL};
|
||||
static const char *defaultModules[] = {
|
||||
"libhtsswf.so.1", "libhtsjava.so.2", "httrack-plugin", NULL
|
||||
};
|
||||
#endif
|
||||
httrackp *opt = malloc(sizeof(httrackp));
|
||||
|
||||
|
||||
359
src/htsname.c
359
src/htsname.c
@@ -41,10 +41,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htstools.h"
|
||||
#include "htscharset.h"
|
||||
#include "htsencoding.h"
|
||||
#include "htssniff.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
#include <ctype.h>
|
||||
|
||||
#define ADD_STANDARD_PATH \
|
||||
@@ -74,6 +70,31 @@ static const char *hts_tbdev[] = {
|
||||
""
|
||||
};
|
||||
|
||||
#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
|
||||
int prev = opt->state._hts_in_html_parsing; \
|
||||
while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
|
||||
opt->state. _hts_in_html_parsing = 6; \
|
||||
/* Wait .. */ \
|
||||
back_wait(sback,opt,cache,0); \
|
||||
/* Transfer rate */ \
|
||||
engine_stats(); \
|
||||
/* Refresh various stats */ \
|
||||
HTS_STAT.stat_nsocket=back_nsoc(sback); \
|
||||
HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
|
||||
HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
|
||||
HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
|
||||
HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
|
||||
HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
|
||||
/* Check */ \
|
||||
{ \
|
||||
if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
|
||||
return -1; \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
opt->state._hts_in_html_parsing = prev; \
|
||||
} while(0)
|
||||
|
||||
/* Strip all // */
|
||||
static void cleanDoubleSlash(char *s) {
|
||||
int i, j;
|
||||
@@ -117,191 +138,37 @@ static void cleanEndingSpaceOrDot(char *s) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Wire Content-Type vs URL extension: a patchable wire type wins over an
|
||||
unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext
|
||||
(#267 guard), a declared disagreement is CONTESTED (sniffed below). */
|
||||
typedef enum wire_verdict {
|
||||
WIRE_KEEPS_EXT,
|
||||
WIRE_WINS,
|
||||
WIRE_CONTESTED
|
||||
} wire_verdict;
|
||||
|
||||
static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime,
|
||||
const char *file, char *urlmime,
|
||||
size_t urlmime_size) {
|
||||
if (may_unknown2(opt, wiremime, file))
|
||||
return WIRE_KEEPS_EXT; /* type kept verbatim (keep-list / bogus-multiple) */
|
||||
urlmime[0] = '\0';
|
||||
/* type implied by the URL extension, only when confidently known (flag 0) */
|
||||
if (!get_httptype_sized(opt, urlmime, urlmime_size, file, 0))
|
||||
return WIRE_WINS; /* URL ext implies no known type */
|
||||
if (strfield2(wiremime, urlmime))
|
||||
return WIRE_KEEPS_EXT; /* agreement (no .htm->.html churn) */
|
||||
if (!is_hypertext_mime(opt, urlmime, file) &&
|
||||
strfield2(wiremime, HTS_UNKNOWN_MIME))
|
||||
return WIRE_KEEPS_EXT; /* no declared type */
|
||||
return WIRE_CONTESTED;
|
||||
}
|
||||
|
||||
/* Optional evidence for a contested wire-vs-ext verdict. */
|
||||
typedef struct sniff_src {
|
||||
struct_back *sback; /* live backing (looked up by adr/fil) */
|
||||
const lien_back *headers; /* snapshot: r.adr, else the url_sav file */
|
||||
const char *adr, *fil;
|
||||
const char *prev_save; /* previous run's save name (cache X-Save) */
|
||||
} sniff_src;
|
||||
|
||||
#if HTS_USEZLIB
|
||||
/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */
|
||||
static size_t sniff_inflate_head(const void *in, size_t in_len, void *out,
|
||||
size_t out_len) {
|
||||
z_stream zs;
|
||||
size_t n = 0;
|
||||
int err;
|
||||
|
||||
memset(&zs, 0, sizeof(zs));
|
||||
if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
|
||||
return 0;
|
||||
zs.next_in = (const Bytef *) in;
|
||||
zs.avail_in = (uInt) in_len;
|
||||
zs.next_out = (Bytef *) out;
|
||||
zs.avail_out = (uInt) out_len;
|
||||
err = inflate(&zs, Z_SYNC_FLUSH);
|
||||
if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
|
||||
n = out_len - zs.avail_out;
|
||||
inflateEnd(&zs);
|
||||
return n;
|
||||
}
|
||||
#endif
|
||||
|
||||
static size_t sniff_read_head(const char *path, void *buf, size_t len) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb");
|
||||
size_t n = 0;
|
||||
|
||||
if (fp != NULL) {
|
||||
n = fread(buf, 1, len, fp);
|
||||
fclose(fp);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Body head of one slot: memory, else its flushed on-disk file (url_sav, or
|
||||
tmpfile for a compressed stream); inflated so the sniff sees the final body.
|
||||
*/
|
||||
static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) {
|
||||
const htsblk *const r = &slot->r;
|
||||
size_t n = 0;
|
||||
|
||||
if (r->adr != NULL && r->size > 0) {
|
||||
n = (size_t) r->size < len ? (size_t) r->size : len;
|
||||
memcpy(buf, r->adr, n);
|
||||
} else {
|
||||
if (r->out != NULL)
|
||||
fflush(r->out);
|
||||
if (slot->url_sav[0] != '\0')
|
||||
n = sniff_read_head(slot->url_sav, buf, len);
|
||||
if (n == 0 && slot->tmpfile != NULL && slot->tmpfile[0] != '\0')
|
||||
n = sniff_read_head(slot->tmpfile, buf, len);
|
||||
}
|
||||
if (n > 0 && r->compressed) {
|
||||
#if HTS_USEZLIB
|
||||
unsigned char raw[HTS_SNIFF_LEN];
|
||||
|
||||
if (n > sizeof(raw))
|
||||
n = sizeof(raw);
|
||||
memcpy(raw, buf, n);
|
||||
n = sniff_inflate_head(raw, n, buf, len);
|
||||
#else
|
||||
n = 0;
|
||||
#endif
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Up to len leading body bytes; 0 when unavailable, and always in
|
||||
non-delayed mode (its HEAD-probe first run couldn't sniff either). */
|
||||
static size_t sniff_body_head(httrackp *opt, const sniff_src *src, void *buf,
|
||||
size_t len) {
|
||||
size_t n = 0;
|
||||
|
||||
if (src == NULL || opt->savename_delayed == HTS_SAVENAME_DELAYED_NONE)
|
||||
return 0;
|
||||
/* live backing slot: a snapshot (back_copy_static) loses r.adr/r.out */
|
||||
if (src->sback != NULL && src->adr != NULL && src->fil != NULL) {
|
||||
const int b = back_index(opt, src->sback, src->adr, src->fil, NULL);
|
||||
|
||||
if (b >= 0)
|
||||
n = sniff_slot_head(&src->sback->lnk[b], buf, len);
|
||||
}
|
||||
if (n == 0 && src->headers != NULL)
|
||||
n = sniff_slot_head(src->headers, buf, len);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Contested verdicts: magic proving the URL ext keeps it, else wire wins. */
|
||||
static int wire_patches_ext(httrackp *opt, const sniff_src *src,
|
||||
const char *wiremime, const char *file) {
|
||||
/* Should the wire Content-Type override the URL's own extension when naming the
|
||||
saved file? True when the type is patchable (may_unknown2) and either the URL
|
||||
extension implies no specific type or the server declared a disagreeing one.
|
||||
A URL extension mapping to a specific non-HTML type is kept only when the
|
||||
server declared NO type (the HTS_UNKNOWN_MIME sentinel; the #267 mangle
|
||||
guard): a typeless .png stays .png, but a .pdf explicitly served as text/html
|
||||
is named .html. The sentinel rides the cache, so updates stay consistent. */
|
||||
static int wire_patches_ext(httrackp *opt, const char *wiremime,
|
||||
const char *file) {
|
||||
char urlmime[256];
|
||||
|
||||
switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) {
|
||||
case WIRE_KEEPS_EXT:
|
||||
if (may_unknown2(opt, wiremime, file))
|
||||
return 0; /* type kept verbatim (keep-list / bogus-multiple) */
|
||||
urlmime[0] = '\0';
|
||||
/* type implied by the URL extension, only when confidently known (flag 0) */
|
||||
if (!get_httptype_sized(opt, urlmime, sizeof(urlmime), file, 0))
|
||||
return 1; /* URL ext implies no known type: trust the wire type */
|
||||
if (strfield2(wiremime, urlmime))
|
||||
return 0; /* wire agrees with the ext: keep it (no .htm->.html churn) */
|
||||
/* wire disagrees with a specific non-HTML URL ext. Keep the ext only when
|
||||
the server declared no type (the sentinel); an explicitly declared type,
|
||||
even text/html, is trusted, so a binary-looking URL that really serves
|
||||
HTML (login/error interstitial, soft-404) is named .html. */
|
||||
if (!is_hypertext_mime(opt, urlmime, file) &&
|
||||
strfield2(wiremime, HTS_UNKNOWN_MIME))
|
||||
return 0;
|
||||
case WIRE_WINS:
|
||||
return 1;
|
||||
case WIRE_CONTESTED:
|
||||
break;
|
||||
}
|
||||
if (src != NULL) {
|
||||
if (hts_sniff_mime_known(urlmime)) {
|
||||
unsigned char head[HTS_SNIFF_LEN];
|
||||
const size_t n = sniff_body_head(opt, src, head, sizeof(head));
|
||||
|
||||
if (n > 0)
|
||||
return hts_sniff_mime_consistent(head, n, urlmime) ? 0 : 1;
|
||||
}
|
||||
/* no bytes: reproduce the previous run's verdict (cached X-Save name) */
|
||||
if (src->prev_save != NULL && src->prev_save[0] != '\0') {
|
||||
char prevmime[256];
|
||||
|
||||
prevmime[0] = '\0';
|
||||
if (get_httptype_sized(opt, prevmime, sizeof(prevmime), src->prev_save,
|
||||
0) &&
|
||||
strfield2(prevmime, urlmime))
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime,
|
||||
const char *file) {
|
||||
char urlmime[256];
|
||||
|
||||
return wiremime != NULL && strnotempty(wiremime) &&
|
||||
wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime)) ==
|
||||
WIRE_CONTESTED &&
|
||||
hts_sniff_mime_known(urlmime);
|
||||
}
|
||||
|
||||
/* Wire-metadata name change: a Content-Disposition filename wins (returns 2),
|
||||
else the declared type's ext when wire_patches_ext() allows (returns 1),
|
||||
else 0. ext receives the new extension or replacement filename. */
|
||||
static int resolve_extension(httrackp *opt, const sniff_src *src,
|
||||
const char *cdispo, const char *contenttype,
|
||||
const char *fil, char *ext, size_t ext_size) {
|
||||
if (strnotempty(cdispo)) {
|
||||
strlcpybuff(ext, cdispo, ext_size);
|
||||
return 2;
|
||||
}
|
||||
if (wire_patches_ext(opt, src, contenttype, fil) &&
|
||||
give_mimext(ext, ext_size, contenttype))
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Build the local save name (save) from adr/fil; renames on collision
|
||||
// (e.g. INDEX.HTML vs index.html).
|
||||
// forme le nom du fichier à sauver (save) à partir de fil et adr
|
||||
// système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html)
|
||||
int url_savename(lien_adrfilsave *const afs,
|
||||
lien_adrfil *const former,
|
||||
const char *referer_adr, const char *referer_fil,
|
||||
@@ -538,30 +405,45 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
// si option check_type activée
|
||||
if (is_html < 0 && opt->check_type && !ext_chg) {
|
||||
int ishtest = 0;
|
||||
|
||||
if (protocol != PROTOCOL_FILE
|
||||
&& protocol != PROTOCOL_FTP
|
||||
) {
|
||||
// tester type avec requète HEAD si on ne connait pas le type du fichier
|
||||
if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/'))) // slash doit être html?
|
||||
if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
|
||||
ishtml(opt, fil) < 0) { // unsure whether it's html or a file
|
||||
(ishtest = ishtml(opt, fil)) <
|
||||
0) { // unsure whether it's html or a file
|
||||
// lire dans le cache
|
||||
char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
|
||||
htsblk r;
|
||||
htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
|
||||
|
||||
previous_save[0] = '\0';
|
||||
r = cache_read_including_broken(opt, cache, adr, fil,
|
||||
previous_save); // test uniquement
|
||||
if (r.statuscode != -1) { // pas d'erreur de lecture cache
|
||||
char s[32];
|
||||
|
||||
if (r.statuscode != -1) { // cache entry read OK
|
||||
s[0] = '\0';
|
||||
hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s",
|
||||
adr_complete, fil_complete);
|
||||
if (!HTTP_IS_REDIRECT(r.statuscode)) {
|
||||
const sniff_src src = {sback, NULL, adr, fil, previous_save};
|
||||
|
||||
ext_chg = resolve_extension(opt, &src, r.cdispo, r.contenttype,
|
||||
fil, ext, sizeof(ext));
|
||||
if (strnotempty(r.cdispo)) { /* filename given */
|
||||
ext_chg = 2; /* change filename */
|
||||
strcpybuff(ext, r.cdispo);
|
||||
} else if (wire_patches_ext(opt, r.contenttype, fil)) {
|
||||
if (give_mimext(s, sizeof(s),
|
||||
r.contenttype)) { // recognized extension
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifdef DEFAULT_BIN_EXT
|
||||
// no extension and potentially bogus
|
||||
else if (ishtest == -2) {
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, DEFAULT_BIN_EXT + 1);
|
||||
}
|
||||
#endif
|
||||
//
|
||||
} else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD &&
|
||||
is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
|
||||
Lookup mimetype not only by extension,
|
||||
@@ -585,13 +467,22 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// fail later
|
||||
else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
|
||||
!opt->state.stop) {
|
||||
// Check if the file is ready in backing.
|
||||
// Check if the file is ready in backing. We basically take the same logic as later.
|
||||
// FIXME: we should cleanup and factorize this unholy mess
|
||||
if (headers != NULL && headers->status >= 0 && !is_redirect) {
|
||||
const sniff_src src = {sback, headers, adr, fil, NULL};
|
||||
|
||||
ext_chg = resolve_extension(opt, &src, headers->r.cdispo,
|
||||
headers->r.contenttype,
|
||||
headers->url_fil, ext, sizeof(ext));
|
||||
if (strnotempty(headers->r.cdispo)) { /* filename given */
|
||||
ext_chg = 2; /* change filename */
|
||||
strcpybuff(ext, headers->r.cdispo);
|
||||
} else if (wire_patches_ext(opt, headers->r.contenttype,
|
||||
headers->url_fil)) {
|
||||
char s[16];
|
||||
if (give_mimext(
|
||||
s, sizeof(s),
|
||||
headers->r.contenttype)) { // recognized extension
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (mime_type != NULL) {
|
||||
ext[0] = '\0';
|
||||
@@ -609,6 +500,13 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (!may_unknown2(opt, mime_type, fil)) {
|
||||
ext_chg = 1;
|
||||
}
|
||||
#ifdef DEFAULT_BIN_EXT
|
||||
// no extension and potentially bogus
|
||||
else if (ishtml(opt, fil) == -2) {
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, DEFAULT_BIN_EXT + 1);
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
ext_chg = 0;
|
||||
}
|
||||
@@ -627,10 +525,11 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
int has_been_moved = 0;
|
||||
lien_adrfil current;
|
||||
|
||||
/* Wait for an available test slot, honoring the connection limits
|
||||
/* Ensure we don't use too many sockets by using a "testing" one
|
||||
If we have only 1 simultaneous connection authorized, wait for pending download
|
||||
Wait for an available slot
|
||||
*/
|
||||
if (!hts_wait_available_socket(sback, opt, cache, ptr))
|
||||
return -1;
|
||||
URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
|
||||
|
||||
/* Rock'in */
|
||||
current.adr[0] = current.fil[0] = '\0';
|
||||
@@ -660,11 +559,24 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (ptr >= 0) {
|
||||
back_fillmax(sback, opt, cache, ptr, numero_passe);
|
||||
}
|
||||
if (!hts_loop_tick(sback, opt, b, ptr)) {
|
||||
// on est obligé d'appeler le shell pour le refresh..
|
||||
// Transfer rate
|
||||
engine_stats();
|
||||
|
||||
// Refresh various stats
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
|
||||
if (!RUN_CALLBACK7
|
||||
(opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart),
|
||||
&HTS_STAT)) {
|
||||
return -1;
|
||||
} else if (opt->state._hts_cancel ||
|
||||
!back_checkmirror(
|
||||
opt)) { // cancel level 2 or 1 (cancel parsing)
|
||||
} else if (opt->state._hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing)
|
||||
back_delete(opt, cache, sback, b); // cancel test
|
||||
stop_looping = 1;
|
||||
}
|
||||
@@ -729,9 +641,8 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
"Loop with HEAD request (during prefetch) at %s%s",
|
||||
current.adr, current.fil);
|
||||
}
|
||||
if (!hts_wait_available_socket(sback, opt,
|
||||
cache, ptr))
|
||||
return -1;
|
||||
// Ajouter
|
||||
URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
|
||||
if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) { // OK
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"(during prefetch) %s (%d) to link %s at %s%s",
|
||||
@@ -785,10 +696,30 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// libérer emplacement backing
|
||||
}
|
||||
|
||||
// no error: change the type?
|
||||
ext_chg = resolve_extension(
|
||||
opt, NULL, back[b].r.cdispo, back[b].r.contenttype,
|
||||
back[b].url_fil, ext, sizeof(ext));
|
||||
{ // pas d'erreur, changer type?
|
||||
char s[16];
|
||||
|
||||
s[0] = '\0';
|
||||
if (strnotempty(back[b].r.cdispo)) { /* filename given */
|
||||
ext_chg = 2; /* change filename */
|
||||
strcpybuff(ext, back[b].r.cdispo);
|
||||
} else if (wire_patches_ext(opt, back[b].r.contenttype,
|
||||
back[b].url_fil)) {
|
||||
if (give_mimext(
|
||||
s, sizeof(s),
|
||||
back[b].r.contenttype)) { // recognized extension
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, s);
|
||||
}
|
||||
}
|
||||
#ifdef DEFAULT_BIN_EXT
|
||||
// no extension and potentially bogus
|
||||
else if (ishtest == -2) {
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, DEFAULT_BIN_EXT + 1);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
// FIN Si non déplacé, forcer type?
|
||||
|
||||
|
||||
@@ -100,8 +100,6 @@ void standard_name(char *b, size_t bsize, const char *dot_pos,
|
||||
const char *nom_pos, const char *fil_complete,
|
||||
int short_ver);
|
||||
void url_savename_addstr(char *d, const char *s);
|
||||
/* Contested wire-vs-ext verdict that a body sniff could settle (htssniff.h). */
|
||||
int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, const char *file);
|
||||
char *url_md5(char *digest_buffer, const char *fil_complete);
|
||||
void url_savename_refname(const char *adr, const char *fil, char *filename);
|
||||
char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
|
||||
|
||||
484
src/htsparse.c
484
src/htsparse.c
@@ -49,7 +49,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsindex.h"
|
||||
#include "htscharset.h"
|
||||
#include "htsencoding.h"
|
||||
#include "htssniff.h"
|
||||
|
||||
/* external modules */
|
||||
#include "htsmodules.h"
|
||||
@@ -78,14 +77,13 @@ Please visit our Website: http://www.httrack.com
|
||||
/** Append to the output buffer the string 'A'. **/
|
||||
#define HT_ADD(A) TypedArrayAppend(output_buffer, A, strlen(A))
|
||||
|
||||
/* clang-format off: an edit realigns all backslashes, churning the macro. */
|
||||
/* clang-format off */
|
||||
/** Append 'A' to the output buffer, html-escaped; FACTOR = max byte expansion. **/
|
||||
#define HT_ADD_HTMLESCAPED_ANY(A, FUNCTION, FACTOR) do { \
|
||||
/** Append to the output buffer the string 'A', html-escaped. **/
|
||||
#define HT_ADD_HTMLESCAPED_ANY(A, FUNCTION) do { \
|
||||
if ((opt->getmode & 1) != 0 && ptr>0) { \
|
||||
const char *const str_ = (A); \
|
||||
size_t size_; \
|
||||
TypedArrayEnsureRoom(output_buffer, strlen(str_) * (FACTOR) + 1024); \
|
||||
/* & is the maximum expansion */ \
|
||||
TypedArrayEnsureRoom(output_buffer, strlen(str_) * 5 + 1024); \
|
||||
size_ = FUNCTION(str_, &TypedArrayTail(output_buffer), \
|
||||
TypedArrayRoom(output_buffer)); \
|
||||
TypedArraySize(output_buffer) += size_; \
|
||||
@@ -93,113 +91,188 @@ Please visit our Website: http://www.httrack.com
|
||||
} while(0)
|
||||
|
||||
/** Append to the output buffer the string 'A', html-escaped for &. **/
|
||||
#define HT_ADD_HTMLESCAPED(A) \
|
||||
HT_ADD_HTMLESCAPED_ANY(A, escape_for_html_print, HTS_HTMLESCAPE_MAXEXP)
|
||||
#define HT_ADD_HTMLESCAPED(A) HT_ADD_HTMLESCAPED_ANY(A, escape_for_html_print)
|
||||
|
||||
/**
|
||||
* Append to the output buffer the string 'A', html-escaped for & and
|
||||
* Append to the output buffer the string 'A', html-escaped for & and
|
||||
* high chars.
|
||||
**/
|
||||
#define HT_ADD_HTMLESCAPED_FULL(A) \
|
||||
HT_ADD_HTMLESCAPED_ANY(A, escape_for_html_print_full, HTS_HTMLESCAPE_FULL_MAXEXP)
|
||||
/* clang-format on */
|
||||
#define HT_ADD_HTMLESCAPED_FULL(A) HT_ADD_HTMLESCAPED_ANY(A, escape_for_html_print_full)
|
||||
|
||||
// does nothing
|
||||
#define XH_uninit do {} while(0)
|
||||
|
||||
#define HT_ADD_END { \
|
||||
int ok=0;\
|
||||
if (TypedArraySize(output_buffer) != 0) { \
|
||||
const size_t ht_len = TypedArraySize(output_buffer); \
|
||||
const char *const ht_buff = TypedArrayElts(output_buffer); \
|
||||
char digest[32+2];\
|
||||
off_t fsize_old = fsize(fconv(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),savename()));\
|
||||
digest[0] = '\0';\
|
||||
domd5mem(TypedArrayElts(output_buffer), ht_len, digest, 1);\
|
||||
if (fsize_old == (off_t) ht_len) { \
|
||||
int mlen = 0;\
|
||||
char* mbuff;\
|
||||
cache_readdata(cache,"//[HTML-MD5]//",savename(),&mbuff,&mlen);\
|
||||
if (mlen) \
|
||||
mbuff[mlen]='\0';\
|
||||
if ((mlen == 32) && (strcmp(((mbuff!=NULL)?mbuff:""),digest)==0)) {\
|
||||
ok=1;\
|
||||
hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s",savename());\
|
||||
} else {\
|
||||
ok=0;\
|
||||
} \
|
||||
}\
|
||||
if (!ok) { \
|
||||
file_notify(opt,urladr(), urlfil(), savename(), 1, 1, r->notmodified); \
|
||||
fp=filecreate(&opt->state.strc, savename()); \
|
||||
if (fp) { \
|
||||
if (ht_len>0) {\
|
||||
if (fwrite(ht_buff,1,ht_len,fp) != ht_len) { \
|
||||
int fcheck;\
|
||||
if ((fcheck=check_fatal_io_errno())) {\
|
||||
opt->state.exit_xh=-1;\
|
||||
}\
|
||||
if (opt->log) { \
|
||||
hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to write HTML file %s", savename());\
|
||||
if (fcheck) {\
|
||||
hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");\
|
||||
}\
|
||||
}\
|
||||
}\
|
||||
}\
|
||||
fclose(fp); fp=NULL; \
|
||||
if (strnotempty(r->lastmodified)) \
|
||||
set_filetime_rfc822(savename(),r->lastmodified); \
|
||||
} else {\
|
||||
int fcheck;\
|
||||
if ((fcheck=check_fatal_io_errno())) {\
|
||||
hts_log_print(opt, LOG_ERROR, "Mirror aborted: disk full or filesystem problems"); \
|
||||
opt->state.exit_xh=-1;\
|
||||
}\
|
||||
hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to save file %s", savename());\
|
||||
if (fcheck) {\
|
||||
hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");\
|
||||
}\
|
||||
}\
|
||||
} else {\
|
||||
file_notify(opt,urladr(), urlfil(), savename(), 0, 0, r->notmodified); \
|
||||
filenote(&opt->state.strc, savename(),NULL); \
|
||||
}\
|
||||
if (cache->ndx)\
|
||||
cache_writedata(cache->ndx,cache->dat,"//[HTML-MD5]//",savename(),digest,(int)strlen(digest));\
|
||||
} \
|
||||
TypedArrayFree(output_buffer); \
|
||||
}
|
||||
#define HT_ADD_FOP
|
||||
|
||||
/* Mutable extended-context fields: one source of truth so the DEFINE/SET/SAVE
|
||||
load and store lists can't drift apart. */
|
||||
/* clang-format off */
|
||||
#define ENGINE_MUTABLE_FIELDS(X) \
|
||||
X(int, error, stre->error_) \
|
||||
X(int, store_errpage, stre->store_errpage_) \
|
||||
X(int, makeindex_done, stre->makeindex_done_) \
|
||||
X(FILE *, makeindex_fp, stre->makeindex_fp_) \
|
||||
X(int, makeindex_links, stre->makeindex_links_) \
|
||||
X(LLint, stat_fragment, stre->stat_fragment_)
|
||||
|
||||
#define ENGINE_FIELD_DECLARE(type, name, src) type name = *(src);
|
||||
#define ENGINE_FIELD_LOAD(type, name, src) name = *(src);
|
||||
#define ENGINE_FIELD_STORE(type, name, src) *(src) = name;
|
||||
// COPY IN HTSCORE.C
|
||||
#define HT_INDEX_END do { \
|
||||
if (!makeindex_done) { \
|
||||
if (makeindex_fp) { \
|
||||
char BIGSTK tempo[1024]; \
|
||||
if (makeindex_links == 1) { \
|
||||
char BIGSTK link_escaped[HTS_URLMAXSIZE*2]; \
|
||||
escape_uri_utf(makeindex_firstlink, link_escaped, sizeof(link_escaped)); \
|
||||
snprintf(tempo,sizeof(tempo),"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">"CRLF,link_escaped); \
|
||||
} else \
|
||||
tempo[0]='\0'; \
|
||||
hts_template_format(makeindex_fp,template_footer, \
|
||||
"<!-- Mirror and index made by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->", \
|
||||
tempo, /* EOF */ NULL \
|
||||
); \
|
||||
fflush(makeindex_fp); \
|
||||
fclose(makeindex_fp); /* à ne pas oublier sinon on passe une nuit blanche */ \
|
||||
makeindex_fp=NULL; \
|
||||
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html_utf8),"index.html"),"primary","primary"); \
|
||||
} \
|
||||
} \
|
||||
makeindex_done=1; /* ok c'est fait */ \
|
||||
} while(0)
|
||||
|
||||
#define ENGINE_DEFINE_CONTEXT() \
|
||||
ENGINE_DEFINE_CONTEXT_BASE(); \
|
||||
/* */ \
|
||||
htsblk* const r HTS_UNUSED = stre->r_; \
|
||||
hash_struct* const hash HTS_UNUSED = stre->hash_; \
|
||||
char* const codebase HTS_UNUSED = stre->codebase; \
|
||||
char* const base HTS_UNUSED = stre->base; \
|
||||
/* */ \
|
||||
const char * const template_header HTS_UNUSED = stre->template_header_; \
|
||||
const char * const template_body HTS_UNUSED = stre->template_body_; \
|
||||
const char * const template_footer HTS_UNUSED = stre->template_footer_; \
|
||||
/* */ \
|
||||
HTS_UNUSED char* const makeindex_firstlink = stre->makeindex_firstlink_; \
|
||||
ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_DECLARE) \
|
||||
/* load-once (kept out of SET/SAVE): re-reading would reset the throttle */ \
|
||||
/* */ \
|
||||
/* */ \
|
||||
int error = * stre->error_; \
|
||||
int store_errpage = * stre->store_errpage_; \
|
||||
/* */ \
|
||||
int makeindex_done = *stre->makeindex_done_; \
|
||||
FILE* makeindex_fp = *stre->makeindex_fp_; \
|
||||
int makeindex_links = *stre->makeindex_links_; \
|
||||
/* */ \
|
||||
LLint stat_fragment = *stre->stat_fragment_; \
|
||||
HTS_UNUSED TStamp makestat_time = stre->makestat_time; \
|
||||
HTS_UNUSED FILE* makestat_fp = stre->makestat_fp
|
||||
|
||||
#define ENGINE_SET_CONTEXT() \
|
||||
ENGINE_SET_CONTEXT_BASE(); \
|
||||
ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_LOAD)
|
||||
/* */ \
|
||||
error = * stre->error_; \
|
||||
store_errpage = * stre->store_errpage_; \
|
||||
/* */ \
|
||||
makeindex_done = *stre->makeindex_done_; \
|
||||
makeindex_fp = *stre->makeindex_fp_; \
|
||||
makeindex_links = *stre->makeindex_links_; \
|
||||
/* */ \
|
||||
stat_fragment = *stre->stat_fragment_; \
|
||||
makestat_time = stre->makestat_time; \
|
||||
makestat_fp = stre->makestat_fp
|
||||
|
||||
#define ENGINE_LOAD_CONTEXT() \
|
||||
ENGINE_DEFINE_CONTEXT()
|
||||
|
||||
#define ENGINE_SAVE_CONTEXT() \
|
||||
ENGINE_SAVE_CONTEXT_BASE(); \
|
||||
ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_STORE)
|
||||
/* clang-format on */
|
||||
/* */ \
|
||||
* stre->error_ = error; \
|
||||
* stre->store_errpage_ = store_errpage; \
|
||||
/* */ \
|
||||
*stre->makeindex_done_ = makeindex_done; \
|
||||
*stre->makeindex_fp_ = makeindex_fp; \
|
||||
*stre->makeindex_links_ = makeindex_links; \
|
||||
/* */ \
|
||||
*stre->stat_fragment_ = stat_fragment
|
||||
|
||||
#define _FILTERS (*opt->filters.filters)
|
||||
#define _FILTERS_PTR (opt->filters.filptr)
|
||||
#define _ROBOTS ((robots_wizard*)opt->robotsptr)
|
||||
|
||||
/* JS-detection automaton states; INSCRIPT_DEFAULT is the synthetic "any other
|
||||
char" column of the transition table. */
|
||||
typedef enum {
|
||||
INSCRIPT_START = 0,
|
||||
INSCRIPT_ANTISLASH,
|
||||
INSCRIPT_INQUOTE,
|
||||
INSCRIPT_INQUOTE2,
|
||||
INSCRIPT_SLASH,
|
||||
INSCRIPT_SLASHSLASH,
|
||||
INSCRIPT_COMMENT,
|
||||
INSCRIPT_COMMENT2,
|
||||
INSCRIPT_ANTISLASH_IN_QUOTE,
|
||||
INSCRIPT_ANTISLASH_IN_QUOTE2,
|
||||
INSCRIPT_DEFAULT = 256
|
||||
} INSCRIPT;
|
||||
/* Apply current *adr character for the script automate */
|
||||
#define AUTOMATE_LOOKUP_CURRENT_ADR() do { \
|
||||
if (inscript) { \
|
||||
int new_state_pos; \
|
||||
new_state_pos=inscript_state[inscript_state_pos][(unsigned char)*html]; \
|
||||
if (new_state_pos < 0) { \
|
||||
new_state_pos=inscript_state[inscript_state_pos][INSCRIPT_DEFAULT]; \
|
||||
} \
|
||||
assertf(new_state_pos >= 0); \
|
||||
assertf(new_state_pos*sizeof(inscript_state[0]) < sizeof(inscript_state)); \
|
||||
inscript_state_pos=new_state_pos; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define INSCRIPT_NSTATES 10 /* rows in the transition table */
|
||||
|
||||
/* Live view of the parser's automaton locals, set up once so the helpers below
|
||||
can drive it without capturing them by lexical scope. */
|
||||
typedef struct {
|
||||
const int *inscript; /* nonzero while inside a script body */
|
||||
const signed char (*table)[257]; /* [INSCRIPT_NSTATES][257] transitions */
|
||||
INSCRIPT *pos; /* current state */
|
||||
const char **html; /* parse cursor */
|
||||
} script_automate;
|
||||
|
||||
/* Feed the current *html byte to the automaton. No-op outside a script body. */
|
||||
static void hts_automate_lookup(const script_automate *aut) {
|
||||
if (*aut->inscript) {
|
||||
int next = aut->table[*aut->pos][(unsigned char) **aut->html];
|
||||
if (next < 0) {
|
||||
next = aut->table[*aut->pos][INSCRIPT_DEFAULT];
|
||||
}
|
||||
assertf(next >= 0 && next < INSCRIPT_NSTATES);
|
||||
*aut->pos = (INSCRIPT) next;
|
||||
}
|
||||
}
|
||||
|
||||
/* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
|
||||
static void hts_automate_increment(const script_automate *aut, int steps) {
|
||||
while (steps > 0) {
|
||||
(*aut->html)++;
|
||||
hts_automate_lookup(aut);
|
||||
steps--;
|
||||
}
|
||||
}
|
||||
/* Increment current pointer to 'steps' characters, modifying automate if necessary */
|
||||
#define INCREMENT_CURRENT_ADR(steps) do { \
|
||||
int steps__ = (int) ( steps ); \
|
||||
while(steps__ > 0) { \
|
||||
html++; \
|
||||
AUTOMATE_LOOKUP_CURRENT_ADR(); \
|
||||
steps__ --; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* Percent-encode the angle brackets of a string so it is safe to embed inside
|
||||
an HTML comment (the default footer) or any other HTML context. A URL holding
|
||||
@@ -344,7 +417,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
int incomment = 0; // dans un <!--
|
||||
int inscript = 0; // dans un scipt pour applets javascript)
|
||||
int inscript_locked = 0; // in locked script (ie. js file)
|
||||
signed char inscript_state[INSCRIPT_NSTATES][257];
|
||||
signed char inscript_state[10][257];
|
||||
typedef enum {
|
||||
INSCRIPT_START = 0,
|
||||
INSCRIPT_ANTISLASH,
|
||||
INSCRIPT_INQUOTE,
|
||||
INSCRIPT_INQUOTE2,
|
||||
INSCRIPT_SLASH,
|
||||
INSCRIPT_SLASHSLASH,
|
||||
INSCRIPT_COMMENT,
|
||||
INSCRIPT_COMMENT2,
|
||||
INSCRIPT_ANTISLASH_IN_QUOTE,
|
||||
INSCRIPT_ANTISLASH_IN_QUOTE2,
|
||||
INSCRIPT_DEFAULT = 256
|
||||
} INSCRIPT;
|
||||
INSCRIPT inscript_state_pos = INSCRIPT_START;
|
||||
const char *inscript_name = NULL; // script tag name
|
||||
int inscript_tag = 0; // on est dans un <body onLoad="... terminé par >
|
||||
@@ -405,8 +491,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
inscript_state[INSCRIPT_COMMENT2]['*'] = INSCRIPT_COMMENT2;
|
||||
inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE][INSCRIPT_DEFAULT] = INSCRIPT_INQUOTE; /* #8: escape in '' */
|
||||
inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE2][INSCRIPT_DEFAULT] = INSCRIPT_INQUOTE2; /* #9: escape in "" */
|
||||
const script_automate saut = {&inscript, inscript_state,
|
||||
&inscript_state_pos, &html};
|
||||
|
||||
/* Primary list or URLs */
|
||||
if (ptr == 0) {
|
||||
@@ -605,14 +689,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
|
||||
// Decode title with encoding
|
||||
if (str->page_charset_ != NULL &&
|
||||
*str->page_charset_ != '\0') {
|
||||
char *sUtf = hts_convertStringToUTF8(
|
||||
s, strlen(s), str->page_charset_);
|
||||
if (str->page_charset_ != NULL
|
||||
&& *str->page_charset_ != '\0') {
|
||||
char *const sUtf =
|
||||
hts_convertStringToUTF8(s, strlen(s), str->page_charset_);
|
||||
if (sUtf != NULL) {
|
||||
/* UTF-8 can expand past s[]; truncate to fit */
|
||||
snprintf(s, sizeof(s), "%s", sUtf);
|
||||
freet(sUtf);
|
||||
strcpy(s, sUtf);
|
||||
free(sUtf);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,9 +709,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
|
||||
} else if (heap(ptr)->depth < opt->depth) { // on a sauté level1+1 et level1
|
||||
hts_finish_makeindex(opt, &makeindex_done, &makeindex_fp,
|
||||
makeindex_links, makeindex_firstlink,
|
||||
template_footer, "primary", "primary");
|
||||
HT_INDEX_END;
|
||||
}
|
||||
} // if (opt->makeindex)
|
||||
}
|
||||
@@ -846,7 +927,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
|
||||
/* automate */
|
||||
hts_automate_lookup(&saut);
|
||||
AUTOMATE_LOOKUP_CURRENT_ADR();
|
||||
|
||||
// Note:
|
||||
// Certaines pages ne respectent pas le html
|
||||
@@ -1762,7 +1843,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
// sauter espaces
|
||||
// adr+=p;
|
||||
hts_automate_increment(&saut, p);
|
||||
INCREMENT_CURRENT_ADR(p);
|
||||
while((is_space(*html)
|
||||
|| (inscriptgen && html[0] == '\\' && is_space(html[1])
|
||||
)
|
||||
@@ -1777,7 +1858,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
// puis quitter
|
||||
// html++; // sauter les espaces, "" et cie
|
||||
hts_automate_increment(&saut, 1);
|
||||
INCREMENT_CURRENT_ADR(1);
|
||||
}
|
||||
|
||||
/* Stop at \n (LF) if primary links or link lists */
|
||||
@@ -1792,7 +1873,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (*html == '\\') {
|
||||
if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
|
||||
// html+=2; // sauter
|
||||
hts_automate_increment(&saut, 2);
|
||||
INCREMENT_CURRENT_ADR(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1840,7 +1921,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (srcset_p) {
|
||||
while(html < r->adr + r->size
|
||||
&& (is_realspace(*html) || *html == ','))
|
||||
hts_automate_increment(&saut, 1);
|
||||
INCREMENT_CURRENT_ADR(1);
|
||||
}
|
||||
eadr = html;
|
||||
|
||||
@@ -3300,7 +3381,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
|
||||
assertf(eadr - html >= 0); // Should not go back
|
||||
if (eadr > html) {
|
||||
hts_automate_increment(&saut, (int) (eadr - 1 - html));
|
||||
INCREMENT_CURRENT_ADR(eadr - 1 - html);
|
||||
}
|
||||
// adr=eadr-1; // ** sauter
|
||||
|
||||
@@ -3319,8 +3400,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
q++; // skip whitespace and empty candidates
|
||||
if (q < endp && *q != '\0' && *q != ',' && *q != quote
|
||||
&& *q != '<' && *q != '>' && (unsigned char) *q >= 32) {
|
||||
hts_automate_increment(
|
||||
&saut, (int) (q - html)); // keep the automate in sync
|
||||
INCREMENT_CURRENT_ADR(q - html); // keep the automate in sync
|
||||
ok = 1;
|
||||
goto srcset_next;
|
||||
}
|
||||
@@ -3399,7 +3479,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
|
||||
back_fillmax(sback, opt, cache, ptr, numero_passe);
|
||||
|
||||
if (!hts_loop_tick(sback, opt, 0, ptr)) {
|
||||
// Transfer rate
|
||||
engine_stats();
|
||||
|
||||
// Refresh various stats
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
|
||||
if (!RUN_CALLBACK7
|
||||
(opt, loop, sback->lnk, sback->count, 0, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
|
||||
hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
|
||||
*stre->exit_xh_ = 1; // exit requested
|
||||
XH_uninit;
|
||||
@@ -3410,6 +3503,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
nofollow = 1; // moins violent
|
||||
opt->state._hts_cancel = 0;
|
||||
}
|
||||
|
||||
}
|
||||
// refresh the backing system each 2 seconds
|
||||
if (engine_stats()) {
|
||||
@@ -3446,12 +3540,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
|
||||
/* Flush and save to disk */
|
||||
if (TypedArraySize(output_buffer) != 0) {
|
||||
hts_finish_html_file(
|
||||
opt, cache, r, &fp, TypedArrayElts(output_buffer),
|
||||
TypedArraySize(output_buffer), urladr(), urlfil(), savename());
|
||||
}
|
||||
TypedArrayFree(output_buffer);
|
||||
HT_ADD_END; // achever
|
||||
}
|
||||
//
|
||||
//
|
||||
@@ -3476,24 +3565,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Mirror the savename to tell whether a redirect saves to the same file (#159);
|
||||
* contract in htsparse.h. */
|
||||
hts_boolean hts_redirect_same_savefile(httrackp *opt, const char *cur_adr,
|
||||
const char *cur_fil,
|
||||
const char *moved_adr,
|
||||
const char *moved_fil) {
|
||||
const int norm_slash = opt->urlhack && !opt->no_slash_dedup;
|
||||
const int norm_query = opt->urlhack && !opt->no_query_dedup;
|
||||
char BIGSTK n_fil[HTS_URLMAXSIZE * 2], pn_fil[HTS_URLMAXSIZE * 2];
|
||||
|
||||
if (strcasecmp(jump_identification_const(moved_adr),
|
||||
jump_identification_const(cur_adr)) != 0)
|
||||
return HTS_FALSE;
|
||||
fil_normalized_filtered_ex(moved_fil, n_fil, NULL, norm_slash, norm_query);
|
||||
fil_normalized_filtered_ex(cur_fil, pn_fil, NULL, norm_slash, norm_query);
|
||||
return strcasecmp(n_fil, pn_fil) == 0;
|
||||
}
|
||||
|
||||
/*
|
||||
Check 301, 302, .. statuscodes (moved)
|
||||
*/
|
||||
@@ -3539,9 +3610,36 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
if ((reponse =
|
||||
ident_url_relatif(mov_url, urladr(), urlfil(), moved)) >= 0) {
|
||||
int set_prio_to = 0; // pas de priotité fixéd par wizard
|
||||
// A same-file alias redirect must be followed, not stubbed (#159).
|
||||
const hts_boolean same_savefile = hts_redirect_same_savefile(
|
||||
opt, urladr(), urlfil(), moved->adr, moved->fil);
|
||||
|
||||
// check whether URLHack is harmless or not (per the effective
|
||||
// sub-flags)
|
||||
if (opt->urlhack && (!opt->no_www_dedup || !opt->no_slash_dedup ||
|
||||
!opt->no_query_dedup)) {
|
||||
const int norm_host = !opt->no_www_dedup;
|
||||
const int norm_slash = !opt->no_slash_dedup;
|
||||
const int norm_query = !opt->no_query_dedup;
|
||||
char BIGSTK n_adr[HTS_URLMAXSIZE * 2], n_fil[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK pn_adr[HTS_URLMAXSIZE * 2], pn_fil[HTS_URLMAXSIZE * 2];
|
||||
|
||||
strlcpybuff(n_adr,
|
||||
norm_host ? jump_normalized_const(moved->adr)
|
||||
: jump_identification_const(moved->adr),
|
||||
sizeof(n_adr));
|
||||
strlcpybuff(pn_adr,
|
||||
norm_host ? jump_normalized_const(urladr())
|
||||
: jump_identification_const(urladr()),
|
||||
sizeof(pn_adr));
|
||||
fil_normalized_filtered_ex(moved->fil, n_fil, NULL, norm_slash,
|
||||
norm_query);
|
||||
fil_normalized_filtered_ex(urlfil(), pn_fil, NULL, norm_slash,
|
||||
norm_query);
|
||||
if (strcasecmp(n_adr, pn_adr) == 0
|
||||
&& strcasecmp(n_fil, pn_fil) == 0) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Redirected link is identical because of 'URL Hack' option: %s%s and %s%s",
|
||||
urladr(), urlfil(), moved->adr, moved->fil);
|
||||
}
|
||||
}
|
||||
//if (ident_url_absolute(mov_url,moved->adr,moved->fil)!=-1) { // ok URL reconnue
|
||||
// c'est (en gros) la même URL..
|
||||
// si c'est un problème de casse dans le host c'est que le serveur est buggé
|
||||
@@ -3569,17 +3667,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
hts_log_print(opt, LOG_DEBUG, "moved link accepted: %s%s",
|
||||
moved->adr, moved->fil);
|
||||
}
|
||||
} else if (same_savefile) {
|
||||
// A stub would point at itself; follow the redirect instead.
|
||||
if (hts_acceptlink(opt, ptr, moved->adr, moved->fil, NULL, NULL,
|
||||
&set_prio_to, NULL) != 1) {
|
||||
get_it = 1;
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Redirect to a same-file alias, fetching real "
|
||||
"content: %s%s -> %s%s",
|
||||
urladr(), urlfil(), moved->adr, moved->fil);
|
||||
}
|
||||
} /* sinon traité normalement */
|
||||
} /* sinon traité normalement */
|
||||
}
|
||||
|
||||
//if ((strfield2(moved->adr,urladr())!=0) && (strfield2(moved->fil,urlfil())!=0)) { // identique à casse près
|
||||
@@ -3602,11 +3690,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
heap(heap(ptr)->precedent)->adr,
|
||||
heap(heap(ptr)->precedent)->fil, opt,
|
||||
sback, cache, hash, ptr, numero_passe, NULL) != -1) {
|
||||
// Same-file alias: the reserved name is the invalidated source,
|
||||
// so record anyway.
|
||||
if (same_savefile ||
|
||||
hash_read(hash, savedmoved.save, NULL,
|
||||
HASH_STRUCT_FILENAME) < 0) { // n'existe pas déja
|
||||
if (hash_read(hash, savedmoved.save, NULL, HASH_STRUCT_FILENAME) < 0) { // n'existe pas déja
|
||||
// enregistrer lien avec SAV IDENTIQUE
|
||||
if (hts_record_link(opt, moved->adr, moved->fil, heap(ptr)->sav, "", "", NULL)) {
|
||||
// mode test?
|
||||
@@ -3630,6 +3714,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
"moving %s to an existing file %s",
|
||||
heap(ptr)->fil, urlfil());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3946,8 +4031,22 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
|
||||
{
|
||||
back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
|
||||
|
||||
// Transfer rate
|
||||
engine_stats();
|
||||
|
||||
// Refresh various stats
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
|
||||
b = 0;
|
||||
if (!hts_loop_tick(sback, opt, b, ptr) || !back_checkmirror(opt)) {
|
||||
if (!RUN_CALLBACK7
|
||||
(opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)
|
||||
|| !back_checkmirror(opt)) {
|
||||
hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
|
||||
*stre->exit_xh_ = 1; // exit requested
|
||||
XH_uninit;
|
||||
@@ -4049,11 +4148,21 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
|
||||
while(opt->state._hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) { // on fait la pause..
|
||||
opt->state._hts_in_html_parsing = 6;
|
||||
back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
|
||||
/* time limit (-E) exceeded: stop waiting for a socket (#481) */
|
||||
if (!back_checkmirror(opt))
|
||||
break;
|
||||
|
||||
if (!hts_loop_tick(sback, opt, b, ptr)) {
|
||||
// Transfer rate
|
||||
engine_stats();
|
||||
|
||||
// Refresh various stats
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
|
||||
if (!RUN_CALLBACK7
|
||||
(opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
|
||||
hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
|
||||
*stre->exit_xh_ = 1; // exit requested
|
||||
XH_uninit;
|
||||
@@ -4240,12 +4349,26 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
|
||||
freet(s);
|
||||
}
|
||||
|
||||
if (!hts_loop_tick(sback, opt, b, ptr)) {
|
||||
// Transfer rate
|
||||
engine_stats();
|
||||
|
||||
// Refresh various stats
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
|
||||
if (!RUN_CALLBACK7
|
||||
(opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
|
||||
hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
|
||||
*stre->exit_xh_ = 1; // exit requested
|
||||
XH_uninit;
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#if HTS_POLL
|
||||
@@ -4478,9 +4601,10 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
|
||||
continue_loop = 0;
|
||||
|
||||
/* Wait for an available slot */
|
||||
if (!hts_wait_available_socket(sback, opt, cache, ptr))
|
||||
return -1;
|
||||
/*
|
||||
Wait for an available slot
|
||||
*/
|
||||
WAIT_FOR_AVAILABLE_SOCKET();
|
||||
|
||||
/* We can lookup directly in the cache to speedup this mess */
|
||||
if (opt->delayed_cached) {
|
||||
@@ -4626,28 +4750,39 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
if (ptr >= 0) {
|
||||
back_fillmax(sback, opt, cache, ptr, numero_passe);
|
||||
}
|
||||
if (!hts_loop_tick(sback, opt, b, ptr)) {
|
||||
back_set_unlocked(sback, b);
|
||||
return -1;
|
||||
} else if (opt->state._hts_cancel ||
|
||||
!back_checkmirror(
|
||||
opt)) { // cancel level 2 or 1 (cancel parsing)
|
||||
back_delete(opt, cache, sback, b); // cancel test
|
||||
break;
|
||||
// on est obligé d'appeler le shell pour le refresh..
|
||||
{
|
||||
|
||||
// Transfer rate
|
||||
engine_stats();
|
||||
|
||||
// Refresh various stats
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
|
||||
if (!RUN_CALLBACK7
|
||||
(opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
|
||||
return -1;
|
||||
} else if (opt->state._hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing)
|
||||
back_delete(opt, cache, sback, b); // cancel test
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (
|
||||
/* dns/connect/request */
|
||||
(back[b].status >= 99 && back[b].status <= 101) ||
|
||||
/* For redirects, wait for request to be terminated */
|
||||
(HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0) ||
|
||||
/* Same for errors */
|
||||
(HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0) ||
|
||||
/* Contested type: wait for a sniffable body head (or EOF) */
|
||||
(back[b].r.statuscode == HTTP_OK && back[b].status > 0 &&
|
||||
strnotempty(back[b].r.cdispo) == 0 &&
|
||||
back[b].r.size < HTS_SNIFF_LEN &&
|
||||
hts_ext_sniff_wanted(opt, back[b].r.contenttype,
|
||||
back[b].url_fil)));
|
||||
} while(
|
||||
/* dns/connect/request */
|
||||
(back[b].status >= 99 && back[b].status <= 101)
|
||||
||
|
||||
/* For redirects, wait for request to be terminated */
|
||||
(HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0)
|
||||
||
|
||||
/* Same for errors */
|
||||
(HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0)
|
||||
);
|
||||
if (b >= 0) {
|
||||
back_set_unlocked(sback, b); // Unlocked entry
|
||||
}
|
||||
@@ -4782,9 +4917,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
|
||||
/* Still have a back reference */
|
||||
if (b >= 0) {
|
||||
/* patch url_sav BEFORE finalize: it records/caches under this name
|
||||
*/
|
||||
strcpybuff(back[b].url_sav, afs->save);
|
||||
/* Finalize now as we have the type */
|
||||
if (back[b].status == STATUS_READY) {
|
||||
if (!back[b].finalized) {
|
||||
@@ -4792,6 +4924,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
back_finalize(opt, cache, sback, b);
|
||||
}
|
||||
}
|
||||
/* Patch destination filename for direct-to-disk mode */
|
||||
strcpybuff(back[b].url_sav, afs->save);
|
||||
}
|
||||
|
||||
} // b >= 0
|
||||
|
||||
@@ -116,19 +116,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
|
||||
int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
htsmoduleStructExtended * stre);
|
||||
|
||||
/*
|
||||
Non-zero if a redirect (cur_adr,cur_fil)->(moved_adr,moved_fil) saves to the
|
||||
same local file, so it must be followed rather than turned into a
|
||||
self-pointing "moved" stub (#159). Mirrors the savename: scheme+userinfo
|
||||
stripped, www kept (www dedup is the crawl layer's job), path
|
||||
slash/query-normalized per the URL-hack flags. Not hash_url_equals: that keys
|
||||
on the dedup hash, which folds www and never collapses http<->https.
|
||||
*/
|
||||
hts_boolean hts_redirect_same_savefile(httrackp *opt, const char *cur_adr,
|
||||
const char *cur_fil,
|
||||
const char *moved_adr,
|
||||
const char *moved_fil);
|
||||
|
||||
/*
|
||||
Process user intercations: pause, add link, delete link..
|
||||
*/
|
||||
@@ -175,4 +162,27 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
/* Apply changes */ \
|
||||
* str->ptr_ = ptr
|
||||
|
||||
#define WAIT_FOR_AVAILABLE_SOCKET() do { \
|
||||
int prev = opt->state._hts_in_html_parsing; \
|
||||
while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
|
||||
opt->state._hts_in_html_parsing = 6; \
|
||||
/* Wait .. */ \
|
||||
back_wait(sback,opt,cache,0); \
|
||||
/* Transfer rate */ \
|
||||
engine_stats(); \
|
||||
/* Refresh various stats */ \
|
||||
HTS_STAT.stat_nsocket=back_nsoc(sback); \
|
||||
HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
|
||||
HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
|
||||
HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
|
||||
HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
|
||||
HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
|
||||
/* Check */ \
|
||||
if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, -1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
|
||||
return -1; \
|
||||
} \
|
||||
} \
|
||||
opt->state._hts_in_html_parsing = prev; \
|
||||
} while(0)
|
||||
|
||||
#endif
|
||||
|
||||
@@ -456,13 +456,6 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
|
||||
return b->buf;
|
||||
}
|
||||
|
||||
/** True if 'count' records of >= 1 byte each fit in 'available' bytes; guards
|
||||
an attacker-controlled count driving a large allocation. */
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean hts_count_fits(size_t count,
|
||||
LLint available) {
|
||||
return (available >= 0 && (LLint) count <= available) ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
|
||||
added bounds checking. freet() also NULLs the freed pointer and tolerates
|
||||
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the
|
||||
|
||||
@@ -45,14 +45,11 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htsdefines.h"
|
||||
#include "htslib.h"
|
||||
#include "htsparse.h"
|
||||
#include "htscache_selftest.h"
|
||||
#include "htsdns_selftest.h"
|
||||
#include "htscharset.h"
|
||||
#include "htsencoding.h"
|
||||
#include "htsftp.h"
|
||||
#include "htsmd5.h"
|
||||
#include "htssniff.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
@@ -63,10 +60,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#ifndef _WIN32
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
/* very minimalistic internal tests */
|
||||
static void basic_selftests(void) {
|
||||
@@ -714,8 +707,7 @@ static int st_entities(httrackp *opt, int argc, char **argv) {
|
||||
}
|
||||
s = strdupt(argv[0]);
|
||||
enc = argc >= 2 ? argv[1] : "UTF-8";
|
||||
if (s != NULL &&
|
||||
hts_unescapeEntitiesWithCharset(s, s, strlen(s) + 1, enc) == 0) {
|
||||
if (s != NULL && hts_unescapeEntitiesWithCharset(s, s, strlen(s), enc) == 0) {
|
||||
printf("%s\n", s);
|
||||
freet(s);
|
||||
} else {
|
||||
@@ -724,34 +716,6 @@ static int st_entities(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* The unescapers must reserve one byte for the trailing NUL: a 'max'-byte
|
||||
dest holding 'max' output chars pre-fix wrote dest[max] (1-byte OOB, caught
|
||||
by ASan). Both unescapeEntities and unescapeUrl share the guard. */
|
||||
static int st_unescape_bounds(httrackp *opt, int argc, char **argv) {
|
||||
char dest[4];
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
assertf(hts_unescapeEntities("abcd", dest, sizeof(dest)) == -1);
|
||||
assertf(hts_unescapeUrl("abcd", dest, sizeof(dest)) == -1);
|
||||
assertf(hts_unescapeEntities("abc", dest, sizeof(dest)) == 0);
|
||||
assertf(strcmp(dest, "abc") == 0);
|
||||
/* raw multi-byte UTF-8 flush path (bypasses the per-byte guard) */
|
||||
assertf(hts_unescapeUrl("ab\xC3\xA9", dest, sizeof(dest)) == -1);
|
||||
assertf(hts_unescapeUrl("a\xC3\xA9", dest, sizeof(dest)) == 0);
|
||||
assertf(strcmp(dest, "a\xC3\xA9") == 0);
|
||||
{
|
||||
/* %xx-encoded flush path (utfBufferJ = lastJ rollback) */
|
||||
char wide[8];
|
||||
|
||||
assertf(hts_unescapeUrl("%C3%A9", wide, sizeof(wide)) == 0);
|
||||
assertf(strcmp(wide, "\xC3\xA9") == 0);
|
||||
}
|
||||
printf("unescape-bounds self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_hashtable(httrackp *opt, int argc, char **argv) {
|
||||
char *snum;
|
||||
unsigned long count = 0;
|
||||
@@ -1094,218 +1058,35 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
|
||||
no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
|
||||
registering an already-crawled link (dedup/collision paths). */
|
||||
/* Parse raw response-header lines and print the naming-relevant fields. */
|
||||
static int st_header(httrackp *opt, int argc, char **argv) {
|
||||
htsblk r;
|
||||
int i;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "header: needs at least one raw header line\n");
|
||||
return 1;
|
||||
}
|
||||
memset(&r, 0, sizeof(r));
|
||||
for (i = 0; i < argc; i++) {
|
||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
||||
|
||||
strcpybuff(line, argv[i]);
|
||||
treathead(NULL, "www.example.com", "/", &r, line);
|
||||
}
|
||||
printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
|
||||
static size_t st_decode_body(const char *arg, char *buf, size_t size) {
|
||||
size_t n = 0;
|
||||
|
||||
if (strncmp(arg, "hex:", 4) == 0) {
|
||||
const char *s = arg + 4;
|
||||
|
||||
for (; s[0] != '\0' && s[1] != '\0' && n + 1 < size; s += 2) {
|
||||
unsigned int byte;
|
||||
|
||||
if (sscanf(s, "%2x", &byte) != 1)
|
||||
break;
|
||||
buf[n++] = (char) byte;
|
||||
}
|
||||
} else {
|
||||
n = strlen(arg);
|
||||
if (n >= size)
|
||||
n = size - 1;
|
||||
memcpy(buf, arg, n);
|
||||
}
|
||||
buf[n] = '\0';
|
||||
return n;
|
||||
}
|
||||
|
||||
static int st_sniff(httrackp *opt, int argc, char **argv) {
|
||||
char BIGSTK body[1024];
|
||||
size_t n;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "sniff: needs a content-type and a body\n");
|
||||
return 1;
|
||||
}
|
||||
n = st_decode_body(argv[1], body, sizeof(body));
|
||||
printf("sniff: known=%d consistent=%d\n",
|
||||
hts_sniff_mime_known(argv[0]) == HTS_TRUE,
|
||||
hts_sniff_mime_consistent(body, n, argv[0]) == HTS_TRUE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_savename(httrackp *opt, int argc, char **argv) {
|
||||
lien_adrfilsave afs;
|
||||
cache_back cache;
|
||||
struct_back *sback;
|
||||
hash_struct hash;
|
||||
lien_back headers;
|
||||
const char *adr = "www.example.com";
|
||||
const char *cdispo = NULL;
|
||||
const char *body = NULL;
|
||||
const char *cached = NULL;
|
||||
const char *bodyfile = "st-savename-body.tmp";
|
||||
int statuscode = HTTP_OK, status = 0;
|
||||
int i;
|
||||
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "savename: needs a fil and a content-type\n");
|
||||
return 1;
|
||||
}
|
||||
/* knobs first: hash_init and the prior links depend on them */
|
||||
for (i = 2; i < argc; i++) {
|
||||
const char *const a = argv[i];
|
||||
|
||||
if (strncmp(a, "adr=", 4) == 0)
|
||||
adr = a + 4;
|
||||
else if (strncmp(a, "cdispo=", 7) == 0)
|
||||
cdispo = a + 7;
|
||||
else if (strncmp(a, "statuscode=", 11) == 0)
|
||||
statuscode = atoi(a + 11);
|
||||
else if (strncmp(a, "status=", 7) == 0)
|
||||
status = atoi(a + 7);
|
||||
else if (strncmp(a, "strip=", 6) == 0)
|
||||
StringCopy(opt->strip_query, a + 6);
|
||||
else if (strncmp(a, "urlhack=", 8) == 0)
|
||||
opt->urlhack = atoi(a + 8) ? HTS_TRUE : HTS_FALSE;
|
||||
else if (strncmp(a, "no-www=", 7) == 0)
|
||||
opt->no_www_dedup = atoi(a + 7) ? HTS_TRUE : HTS_FALSE;
|
||||
else if (strncmp(a, "no-slash=", 9) == 0)
|
||||
opt->no_slash_dedup = atoi(a + 9) ? HTS_TRUE : HTS_FALSE;
|
||||
else if (strncmp(a, "no-query=", 9) == 0)
|
||||
opt->no_query_dedup = atoi(a + 9) ? HTS_TRUE : HTS_FALSE;
|
||||
else if (strncmp(a, "n83=", 4) == 0)
|
||||
opt->savename_83 = atoi(a + 4);
|
||||
else if (strncmp(a, "type=", 5) == 0)
|
||||
opt->savename_type = atoi(a + 5);
|
||||
else if (strncmp(a, "body=", 5) == 0)
|
||||
body = a + 5;
|
||||
else if (strncmp(a, "cached=", 7) == 0)
|
||||
cached = a + 7;
|
||||
else if (strncmp(a, "prior=", 6) != 0) {
|
||||
fprintf(stderr, "savename: unknown arg '%s'\n", a);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
memset(&afs, 0, sizeof(afs));
|
||||
strcpybuff(afs.af.adr, adr);
|
||||
strcpybuff(afs.af.adr, "www.example.com");
|
||||
strcpybuff(afs.af.fil, argv[0]);
|
||||
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
if (cached != NULL) { /* cached=<content-type>|<save name> */
|
||||
char *dup = strdupt(cached);
|
||||
char *const sep = strchr(dup, '|');
|
||||
char locbuf[64] = "";
|
||||
htsblk cr;
|
||||
|
||||
if (sep == NULL) {
|
||||
fprintf(stderr, "savename: cached needs ctype|save\n");
|
||||
return 1;
|
||||
}
|
||||
*sep = '\0';
|
||||
/* one-entry cache in cwd, reopened read-only; body is PNG magic on
|
||||
purpose: only the recorded name (X-Save) may drive the naming */
|
||||
StringCopy(opt->path_log, "");
|
||||
cache.type = 1;
|
||||
cache.log = cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache_init(&cache, opt);
|
||||
hts_init_htsblk(&cr);
|
||||
cr.statuscode = HTTP_OK;
|
||||
strcpybuff(cr.msg, "OK");
|
||||
strcpybuff(cr.contenttype, dup);
|
||||
cr.location = locbuf;
|
||||
cr.adr = strdupt("\x89PNG\r\n\x1a\n");
|
||||
cr.size = 8;
|
||||
cache_add(opt, &cache, &cr, adr, argv[0], sep + 1, 1, NULL);
|
||||
freet(cr.adr);
|
||||
if (cache.zipOutput != NULL) {
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
}
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.ro = 1;
|
||||
cache_init(&cache, opt);
|
||||
freet(dup);
|
||||
} else {
|
||||
cache.hashtable = (void *) coucal_new(0);
|
||||
}
|
||||
cache.hashtable = (void *) coucal_new(0);
|
||||
|
||||
sback = back_new(opt, opt->maxsoc * 32 + 1024);
|
||||
/* same wiring as hts_mirror (htscore.c) */
|
||||
hash_init(opt, &hash, opt->urlhack);
|
||||
hash.liens = (const lien_url *const *const *) &opt->liens;
|
||||
opt->hash = &hash;
|
||||
hts_record_init(opt);
|
||||
|
||||
for (i = 2; i < argc; i++) {
|
||||
if (strncmp(argv[i], "prior=", 6) == 0) {
|
||||
char *dup = strdupt(argv[i] + 6);
|
||||
char *const p1 = strchr(dup, '|');
|
||||
char *const p2 = p1 != NULL ? strchr(p1 + 1, '|') : NULL;
|
||||
|
||||
if (p2 == NULL) {
|
||||
fprintf(stderr, "savename: prior needs adr|fil|sav\n");
|
||||
return 1;
|
||||
}
|
||||
*p1 = *p2 = '\0';
|
||||
if (!hts_record_link(opt, dup, p1 + 1, p2 + 1, "", "", NULL))
|
||||
return 1;
|
||||
freet(dup);
|
||||
}
|
||||
}
|
||||
|
||||
memset(&headers, 0, sizeof(headers));
|
||||
headers.status = status;
|
||||
headers.r.statuscode = statuscode;
|
||||
headers.status = 0;
|
||||
headers.r.statuscode = HTTP_OK;
|
||||
strcpybuff(headers.r.contenttype, argv[1]);
|
||||
if (cdispo != NULL)
|
||||
strcpybuff(headers.r.cdispo, cdispo);
|
||||
strcpybuff(headers.url_fil, argv[0]);
|
||||
if (body != NULL) { /* leading body bytes, read via url_sav */
|
||||
char BIGSTK data[1024];
|
||||
const size_t n = st_decode_body(body, data, sizeof(data));
|
||||
FILE *const fp = fopen(bodyfile, "wb");
|
||||
|
||||
if (fp == NULL || fwrite(data, 1, n, fp) != n) {
|
||||
fprintf(stderr, "savename: can not write %s\n", bodyfile);
|
||||
return 1;
|
||||
}
|
||||
fclose(fp);
|
||||
strcpybuff(headers.url_sav, bodyfile);
|
||||
}
|
||||
|
||||
url_savename(&afs, NULL, NULL, NULL, opt, sback, &cache, &hash, 0, 0,
|
||||
&headers);
|
||||
if (body != NULL)
|
||||
(void) UNLINK(bodyfile);
|
||||
printf("savename: %s\n", afs.save);
|
||||
return 0;
|
||||
}
|
||||
@@ -1347,30 +1128,6 @@ static int st_cache_writefail(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
|
||||
int err;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "cache-corrupt: needs a directory\n");
|
||||
return 1;
|
||||
}
|
||||
err = cache_corruption_selftest(opt, argv[0]);
|
||||
printf("cache-corrupt: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_reconcile(httrackp *opt, int argc, char **argv) {
|
||||
int err;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "reconcile: needs a directory\n");
|
||||
return 1;
|
||||
}
|
||||
err = cache_reconcile_selftest(opt, argv[0]);
|
||||
printf("cache-reconcile: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_dns(httrackp *opt, int argc, char **argv) {
|
||||
const int err = dns_selftests(opt);
|
||||
|
||||
@@ -1583,165 +1340,6 @@ static int st_urlhack(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* #159: hts_redirect_same_savefile decides whether a redirect is a same-file
|
||||
* alias. */
|
||||
static int st_redirect_samefile(httrackp *opt, int argc, char **argv) {
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
#define SAME(aa, fa, ab, fb) hts_redirect_same_savefile(opt, aa, fa, ab, fb)
|
||||
/* scheme and userinfo collapse (the #159 case); a different path does not */
|
||||
assertf(SAME("http://foo.com", "/a/b", "https://foo.com", "/a/b"));
|
||||
assertf(SAME("http://user@foo.com", "/a", "http://foo.com", "/a"));
|
||||
assertf(!SAME("http://foo.com", "/a", "http://foo.com", "/b"));
|
||||
/* www stays distinct here; the crawl's dedup layer folds www, not this helper
|
||||
*/
|
||||
opt->urlhack = HTS_TRUE;
|
||||
opt->no_www_dedup = opt->no_slash_dedup = opt->no_query_dedup = HTS_FALSE;
|
||||
assertf(!SAME("http://www.foo.com", "/a", "http://foo.com", "/a"));
|
||||
/* slash/query fold only when the dedup flag is on */
|
||||
assertf(SAME("https://foo.com", "/a//b", "http://foo.com", "/a/b"));
|
||||
assertf(
|
||||
SAME("https://foo.com", "/p?b=2&a=1", "http://foo.com", "/p?a=1&b=2"));
|
||||
opt->no_slash_dedup = opt->no_query_dedup = HTS_TRUE;
|
||||
assertf(!SAME("https://foo.com", "/a//b", "http://foo.com", "/a/b"));
|
||||
assertf(
|
||||
!SAME("https://foo.com", "/p?b=2&a=1", "http://foo.com", "/p?a=1&b=2"));
|
||||
/* but a pure scheme alias still collapses regardless of dedup opt-outs */
|
||||
assertf(SAME("http://foo.com", "/a/b", "https://foo.com", "/a/b"));
|
||||
opt->no_slash_dedup = opt->no_query_dedup = HTS_FALSE;
|
||||
#undef SAME
|
||||
printf("redirect-samefile self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// hts_finish_makeindex writes the footer, emits the refresh meta only when
|
||||
// makeindex_links==1, and clears *fp / sets *done. argv[0] is a writable dir.
|
||||
static int st_makeindex(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
char buf[4096];
|
||||
FILE *fp;
|
||||
size_t n;
|
||||
int done;
|
||||
|
||||
assertf(argc >= 1);
|
||||
snprintf(path, sizeof(path), "%s/index.html", argv[0]);
|
||||
|
||||
/* single first link: footer + a refresh meta carrying the escaped URL */
|
||||
done = 0;
|
||||
fp = fopen(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
hts_finish_makeindex(opt, &done, &fp, 1, "http://example.com/a b", "%s%s", "",
|
||||
"");
|
||||
assertf(fp == NULL); /* the function closed and cleared it */
|
||||
assertf(done != 0);
|
||||
fp = fopen(path, "rb");
|
||||
assertf(fp != NULL);
|
||||
n = fread(buf, 1, sizeof(buf) - 1, fp);
|
||||
fclose(fp);
|
||||
buf[n] = '\0';
|
||||
assertf(strstr(buf, "Mirror and index made by HTTrack") != NULL);
|
||||
assertf(strstr(buf, "Refresh") != NULL);
|
||||
assertf(strstr(buf, "example.com") != NULL);
|
||||
|
||||
/* no single link: footer only, no refresh meta */
|
||||
done = 0;
|
||||
fp = fopen(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
hts_finish_makeindex(opt, &done, &fp, 0, NULL, "%s%s", "", "");
|
||||
assertf(fp == NULL);
|
||||
assertf(done != 0);
|
||||
fp = fopen(path, "rb");
|
||||
assertf(fp != NULL);
|
||||
n = fread(buf, 1, sizeof(buf) - 1, fp);
|
||||
fclose(fp);
|
||||
buf[n] = '\0';
|
||||
assertf(strstr(buf, "Mirror and index made by HTTrack") != NULL);
|
||||
assertf(strstr(buf, "Refresh") == NULL);
|
||||
|
||||
UNLINK(path);
|
||||
printf("makeindex self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Each inplace_escape_*() must equal escape_*() on a copy. */
|
||||
static int st_inplace_escape(httrackp *opt, int argc, char **argv) {
|
||||
/* >255 bytes forces the helper's malloct path, not the stack buffer */
|
||||
static char longstr[600];
|
||||
static const char *const samples[] = {
|
||||
"", "abc", "a b/c?d=e&f", "h\x8ello w\x94rld",
|
||||
"a%b\"c<d>", "/path to/file", longstr};
|
||||
static size_t (*const inplace[])(char *, size_t) = {
|
||||
inplace_escape_in_url, inplace_escape_spc_url, inplace_escape_uri_utf,
|
||||
inplace_escape_check_url, inplace_escape_uri};
|
||||
static size_t (*const plain[])(const char *, char *, size_t) = {
|
||||
escape_in_url, escape_spc_url, escape_uri_utf, escape_check_url,
|
||||
escape_uri};
|
||||
size_t i, f;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
memset(longstr, 'a', sizeof(longstr) - 1);
|
||||
for (f = 0; f < sizeof(inplace) / sizeof(inplace[0]); f++) {
|
||||
for (i = 0; i < sizeof(samples) / sizeof(samples[0]); i++) {
|
||||
char ref[4096], work[4096];
|
||||
size_t rret, iret;
|
||||
rret = plain[f](samples[i], ref, sizeof(ref));
|
||||
strcpybuff(work, samples[i]);
|
||||
iret = inplace[f](work, sizeof(work));
|
||||
assertf(iret == rret);
|
||||
assertf(strcmp(work, ref) == 0);
|
||||
}
|
||||
}
|
||||
printf("inplace-escape self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Pin HTS_HTMLESCAPE*_MAXEXP to each escaper's true max byte expansion. */
|
||||
static int st_escape_room(httrackp *opt, int argc, char **argv) {
|
||||
/* N > 1023: where 6n outgrows the old 5n+1024 reservation */
|
||||
enum { N = 2000 };
|
||||
|
||||
char *src = malloct(N + 1);
|
||||
char *dst;
|
||||
size_t room, got;
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
/* _full worst case: a high byte expands to "&#xHH;" (6 bytes) */
|
||||
memset(src, 0xE9, N);
|
||||
src[N] = '\0';
|
||||
room = (size_t) N * HTS_HTMLESCAPE_FULL_MAXEXP + 1024;
|
||||
dst = malloct(room);
|
||||
got = escape_for_html_print_full(src, dst, room);
|
||||
assertf(got == (size_t) N * HTS_HTMLESCAPE_FULL_MAXEXP);
|
||||
assertf(strlen(dst) == got);
|
||||
freet(dst);
|
||||
|
||||
/* one factor short overflows (returns size), truncating the page: the bug */
|
||||
room = (size_t) N * (HTS_HTMLESCAPE_FULL_MAXEXP - 1) + 1024;
|
||||
dst = malloct(room);
|
||||
got = escape_for_html_print_full(src, dst, room);
|
||||
assertf(got == room);
|
||||
freet(dst);
|
||||
|
||||
/* plain escaper worst case: '&' -> "&" (5); high bytes stay verbatim */
|
||||
memset(src, '&', N);
|
||||
src[N] = '\0';
|
||||
room = (size_t) N * HTS_HTMLESCAPE_MAXEXP + 1024;
|
||||
dst = malloct(room);
|
||||
got = escape_for_html_print(src, dst, room);
|
||||
assertf(got == (size_t) N * HTS_HTMLESCAPE_MAXEXP);
|
||||
assertf(strlen(dst) == got);
|
||||
freet(dst);
|
||||
|
||||
freet(src);
|
||||
printf("escape-room self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Default User-Agent: honest HTTrack token, no resurrected Windows 98. */
|
||||
static int st_useragent(httrackp *opt, int argc, char **argv) {
|
||||
const char *ua = StringBuff(opt->user_agent);
|
||||
@@ -2011,86 +1609,6 @@ static int st_robots(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* get_ftp_line must bound a hostile, CRLF-less reply into its internal
|
||||
1024-byte buffer; ASan turns the pre-fix overflow into an abort here. */
|
||||
#ifndef _WIN32
|
||||
static int st_ftpline(httrackp *opt, int argc, char **argv) {
|
||||
int sv[2];
|
||||
char line[2048];
|
||||
char flood[4096];
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
memset(flood, 'x', sizeof(flood));
|
||||
assertf(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
|
||||
assertf(write(sv[1], "220 ", 4) == 4); // valid 3-digit code
|
||||
assertf(write(sv[1], flood, sizeof(flood)) == (ssize_t) sizeof(flood));
|
||||
assertf(write(sv[1], "\r\n", 2) == 2); // end the line so we return
|
||||
close(sv[1]);
|
||||
line[0] = '\0';
|
||||
get_ftp_line(sv[0], line, sizeof(line), 5);
|
||||
close(sv[0]);
|
||||
printf("ftp-line self-test OK (bounded %d-byte reply)\n",
|
||||
(int) sizeof(flood));
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ftp_split_userpass: well-formed split, plus a hostile over-long userinfo
|
||||
that pre-fix overran user[256]/pass[256]. */
|
||||
static int st_ftpuser(httrackp *opt, int argc, char **argv) {
|
||||
char user[256], pass[256];
|
||||
char in[1200];
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
{
|
||||
const char ok[] = "bob:secret@host/f"; // '@' at index 10
|
||||
|
||||
ftp_split_userpass(ok, ok + 11, user, sizeof(user), pass, sizeof(pass));
|
||||
assertf(strcmp(user, "bob") == 0);
|
||||
assertf(strcmp(pass, "secret") == 0);
|
||||
}
|
||||
memset(in, 'u', 400);
|
||||
in[400] = ':';
|
||||
memset(in + 401, 'p', 400);
|
||||
in[801] = '@';
|
||||
in[802] = '\0';
|
||||
ftp_split_userpass(in, in + 802, user, sizeof(user), pass, sizeof(pass));
|
||||
assertf(strlen(user) == sizeof(user) - 1);
|
||||
assertf(strlen(pass) == sizeof(pass) - 1);
|
||||
{
|
||||
/* tight sizes + guard byte catch an off-by-one the 256 case can't */
|
||||
char ubuf[16], pbuf[16];
|
||||
|
||||
memset(ubuf, 'Z', sizeof(ubuf));
|
||||
memset(pbuf, 'Z', sizeof(pbuf));
|
||||
ftp_split_userpass(in, in + 802, ubuf, 8, pbuf, 8);
|
||||
assertf(strcmp(ubuf, "uuuuuuu") == 0);
|
||||
assertf(strcmp(pbuf, "ppppppp") == 0);
|
||||
assertf(ubuf[8] == 'Z' && pbuf[8] == 'Z');
|
||||
}
|
||||
printf("ftp-userpass self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* hts_count_fits caps the .class constant-pool entry count to the file size,
|
||||
rejecting the ~68 MB-per-file calloc DoS. */
|
||||
static int st_java(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
assertf(hts_count_fits(10, 1000) == HTS_TRUE);
|
||||
assertf(hts_count_fits(0, 10) == HTS_TRUE);
|
||||
assertf(hts_count_fits(65535, 10) == HTS_FALSE);
|
||||
assertf(hts_count_fits(1, 0) == HTS_FALSE);
|
||||
assertf(hts_count_fits(1, -1) == HTS_FALSE);
|
||||
printf("java constant-pool cap self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* Registry: name -> handler, with a usage hint and a one-line description. */
|
||||
/* ------------------------------------------------------------ */
|
||||
@@ -2111,8 +1629,6 @@ static const struct selftest_entry {
|
||||
st_stripquery},
|
||||
{"urlhack", "", "-%u url-hack sub-flag (www/slash/query) self-test",
|
||||
st_urlhack},
|
||||
{"redirect-samefile", "", "same-file redirect detection self-test (#159)",
|
||||
st_redirect_samefile},
|
||||
{"mime", "<filename>", "MIME type for a filename", st_mime},
|
||||
{"charset", "<charset> <string>",
|
||||
"convert a string to UTF-8 from a charset", st_charset},
|
||||
@@ -2121,8 +1637,6 @@ static const struct selftest_entry {
|
||||
{"idna-decode", "<host>", "decode an IDNA/punycode hostname",
|
||||
st_idna_decode},
|
||||
{"entities", "<string> [encoding]", "unescape HTML entities", st_entities},
|
||||
{"unescape-bounds", "", "unescapers reserve the NUL byte (no 1-byte OOB)",
|
||||
st_unescape_bounds},
|
||||
{"hashtable", "<count|file>", "coucal hashtable stress test", st_hashtable},
|
||||
{"strsafe", "[overflow|overflow-buff [str]]", "bounded string-op self-test",
|
||||
st_strsafe},
|
||||
@@ -2132,41 +1646,21 @@ static const struct selftest_entry {
|
||||
st_relative},
|
||||
{"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
|
||||
st_resolve},
|
||||
{"header", "<raw-header-line> ...", "response header-line parsing",
|
||||
st_header},
|
||||
{"savename", "<fil> <content-type> [key=value ...]",
|
||||
"local save-name for a URL", st_savename},
|
||||
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
|
||||
st_sniff},
|
||||
{"savename", "<fil> <content-type>", "local save-name for a URL",
|
||||
st_savename},
|
||||
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
|
||||
{"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
|
||||
st_cache_golden},
|
||||
{"cache-writefail", "<dir>", "cache write-failure handling self-test",
|
||||
st_cache_writefail},
|
||||
{"reconcile", "<dir>", "cache generation reconcile policy self-test",
|
||||
st_reconcile},
|
||||
{"cache-corrupt", "<dir>", "cache read-side corruption self-test",
|
||||
st_cache_corrupt},
|
||||
{"dns", "", "DNS resolver/cache self-test", st_dns},
|
||||
{"cookies", "", "cookie request-header self-test", st_cookies},
|
||||
{"useragent", "", "default User-Agent self-test", st_useragent},
|
||||
{"makeindex", "[dir]", "hts_finish_makeindex footer/refresh self-test",
|
||||
st_makeindex},
|
||||
{"inplace-escape", "", "inplace_escape_* vs escape_* equivalence self-test",
|
||||
st_inplace_escape},
|
||||
{"escape-room", "", "HT_ADD_HTMLESCAPED* reservation-factor self-test",
|
||||
st_escape_room},
|
||||
{"status", "", "HTTP status code -> reason phrase self-test", st_status},
|
||||
{"acceptencoding", "[dir]",
|
||||
"Accept-Encoding advertises gzip+deflate, both decode", st_acceptencoding},
|
||||
{"robots", "", "robots.txt RFC 9309 Allow/Disallow precedence self-test",
|
||||
st_robots},
|
||||
#ifndef _WIN32
|
||||
{"ftp-line", "", "get_ftp_line bounds a hostile FTP reply line",
|
||||
st_ftpline},
|
||||
#endif
|
||||
{"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
|
||||
{"java", "", "java .class constant-pool count cap self-test", st_java},
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
352
src/htssniff.c
352
src/htssniff.c
@@ -1,352 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2017 Xavier Roche and other contributors
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Important notes:
|
||||
|
||||
- We hereby ask people using this source NOT to use it in purpose of grabbing
|
||||
emails addresses, or collecting any other private information on persons.
|
||||
This would disgrace our work, and spoil the many hours we spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: MIME magic-byte consistency checks */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#include "htssniff.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "htslib.h"
|
||||
|
||||
/* One magic rule: `len` bytes at `off` confirm `mime`. */
|
||||
typedef struct sniff_magic {
|
||||
const char *mime;
|
||||
unsigned short off;
|
||||
unsigned char len;
|
||||
const char *bytes;
|
||||
} sniff_magic;
|
||||
|
||||
/* Direction is mime -> magic (verify a claim, never classify); types with
|
||||
no reliable magic (plain text, css, js..) are deliberately absent. Patterns
|
||||
follow the WHATWG MIME Sniffing Standard tables where it defines them
|
||||
(https://mimesniff.spec.whatwg.org/); the rest covers httrack's wider MIME
|
||||
set. Spec-only types absent from our MIME tables (EOT, font/collection)
|
||||
are omitted as unreachable. */
|
||||
static const sniff_magic sniff_table[] = {
|
||||
/* images */
|
||||
{"image/jpeg", 0, 3, "\xff\xd8\xff"},
|
||||
{"image/pipeg", 0, 3, "\xff\xd8\xff"},
|
||||
{"image/pjpeg", 0, 3, "\xff\xd8\xff"},
|
||||
{"image/png", 0, 8, "\x89PNG\r\n\x1a\n"},
|
||||
{"image/gif", 0, 6, "GIF87a"},
|
||||
{"image/gif", 0, 6, "GIF89a"},
|
||||
{"image/bmp", 0, 2, "BM"},
|
||||
{"image/tiff", 0, 4, "II*\0"},
|
||||
{"image/tiff", 0, 4, "MM\0*"},
|
||||
{"image/x-icon", 0, 4, "\0\0\1\0"},
|
||||
{"image/x-icon", 0, 4, "\0\0\2\0"}, /* Windows cursor, per the spec */
|
||||
{"image/x-portable-bitmap", 0, 2, "P1"},
|
||||
{"image/x-portable-bitmap", 0, 2, "P4"},
|
||||
{"image/x-portable-pixmap", 0, 2, "P3"},
|
||||
{"image/x-portable-pixmap", 0, 2, "P6"},
|
||||
{"image/x-xpixmap", 0, 9, "/* XPM */"},
|
||||
{"image/x-xbitmap", 0, 7, "#define"},
|
||||
{"image/x-rgb", 0, 2, "\x01\xda"},
|
||||
{"image/x-cmu-raster", 0, 4, "\xf1\x00\x40\xbb"},
|
||||
/* audio */
|
||||
{"audio/mpeg", 0, 3, "ID3"},
|
||||
{"audio/basic", 0, 4, ".snd"},
|
||||
{"audio/mid", 0, 8, "MThd\0\0\0\6"},
|
||||
{"audio/midi", 0, 8, "MThd\0\0\0\6"},
|
||||
{"audio/x-pn-realaudio", 0, 4, ".ra\xfd"},
|
||||
{"audio/x-pn-realaudio", 0, 4, ".RMF"},
|
||||
{"audio/x-pn-realaudio-plugin", 0, 4, ".ra\xfd"},
|
||||
{"audio/x-pn-realaudio-plugin", 0, 4, ".RMF"},
|
||||
{"audio/flac", 0, 4, "fLaC"},
|
||||
{"audio/aac", 0, 4, "ADIF"},
|
||||
/* video */
|
||||
{"video/mpeg", 0, 4, "\x00\x00\x01\xba"},
|
||||
{"video/mpeg", 0, 4, "\x00\x00\x01\xb3"},
|
||||
{"video/x-sgi-movie", 0, 4, "MOVI"},
|
||||
/* archives / compression */
|
||||
{"application/x-gzip", 0, 3, "\x1f\x8b\x08"},
|
||||
{"multipart/x-gzip", 0, 3, "\x1f\x8b\x08"},
|
||||
{"application/x-compressed", 0, 3, "\x1f\x8b\x08"},
|
||||
{"application/x-compress", 0, 2, "\x1f\x9d"},
|
||||
{"application/x-bzip2", 0, 3, "BZh"},
|
||||
{"application/x-7z-compressed", 0, 6, "7z\xbc\xaf\x27\x1c"},
|
||||
/* 6-byte prefix common to RAR4 (spec) and RAR5 */
|
||||
{"application/x-rar-compressed", 0, 6, "Rar!\x1a\x07"},
|
||||
{"application/zstd", 0, 4, "\x28\xb5\x2f\xfd"},
|
||||
{"application/arj", 0, 2, "\x60\xea"},
|
||||
{"application/x-cpio", 0, 6, "070701"},
|
||||
{"application/x-cpio", 0, 6, "070707"},
|
||||
{"application/x-cpio", 0, 2, "\xc7\x71"},
|
||||
{"application/x-sv4cpio", 0, 6, "070701"},
|
||||
{"application/x-sv4crc", 0, 6, "070702"},
|
||||
{"application/x-stuffit", 0, 8, "StuffIt "},
|
||||
{"application/x-stuffit", 0, 4, "SIT!"},
|
||||
{"application/mac-binhex40", 0, 10, "(This file"},
|
||||
/* documents */
|
||||
{"application/pdf", 0, 5, "%PDF-"},
|
||||
{"application/postscript", 0, 2, "%!"},
|
||||
{"application/rtf", 0, 5, "{\\rtf"},
|
||||
{"application/x-dvi", 0, 2, "\xf7\x02"},
|
||||
{"application/x-hdf", 0, 4, "\x0e\x03\x13\x01"},
|
||||
{"application/x-hdf", 0, 8, "\x89HDF\r\n\x1a\n"},
|
||||
{"application/x-netcdf", 0, 4, "CDF\x01"},
|
||||
{"application/x-netcdf", 0, 4, "CDF\x02"},
|
||||
{"application/x-msaccess", 0, 19, "\0\1\0\0Standard Jet DB"},
|
||||
/* fonts */
|
||||
{"font/woff", 0, 4, "wOFF"},
|
||||
{"font/woff2", 0, 4, "wOF2"},
|
||||
{"font/ttf", 0, 4, "\0\1\0\0"},
|
||||
{"font/ttf", 0, 4, "true"},
|
||||
{"font/otf", 0, 4, "OTTO"},
|
||||
/* misc */
|
||||
{"application/x-shockwave-flash", 0, 3, "FWS"},
|
||||
{"application/x-shockwave-flash", 0, 3, "CWS"},
|
||||
{"application/x-shockwave-flash", 0, 3, "ZWS"},
|
||||
{"application/futuresplash", 0, 3, "FWS"},
|
||||
{"application/x-director", 0, 4, "RIFX"},
|
||||
{"application/x-director", 0, 4, "XFIR"},
|
||||
{"application/x-java-vm", 0, 4, "\xca\xfe\xba\xbe"},
|
||||
{"application/wasm", 0, 4, "\0asm"},
|
||||
{"application/x-msmetafile", 0, 4, "\xd7\xcd\xc6\x9a"},
|
||||
{"application/x-msmetafile", 0, 4, "\x01\x00\x09\x00"},
|
||||
{"application/x-x509-ca-cert", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs12", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs7-mime", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs7-signature", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs7-certificates", 0, 2, "\x30\x82"},
|
||||
{"x-world/x-vrml", 0, 5, "#VRML"},
|
||||
{"application/x-bittorrent", 0, 11, "d8:announce"},
|
||||
{"drawing/x-dwf", 0, 4, "(DWF"},
|
||||
{"application/acad", 0, 4, "AC10"},
|
||||
{NULL, 0, 0, NULL}};
|
||||
|
||||
/* MIME families sharing a container magic */
|
||||
static const char *const zip_mimes[] = {
|
||||
"application/zip", "application/x-zip-compressed", "multipart/x-zip", NULL};
|
||||
static const char *const zip_mime_prefixes[] = {
|
||||
"application/vnd.openxmlformats-officedocument.",
|
||||
"application/vnd.oasis.opendocument.", NULL};
|
||||
static const char *const ole_mimes[] = {"application/msword",
|
||||
"application/excel",
|
||||
"application/vnd.ms-excel",
|
||||
"application/powerpoint",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.ms-project",
|
||||
"application/vnd.ms-works",
|
||||
"application/x-msmoney",
|
||||
"application/x-mspublisher",
|
||||
NULL};
|
||||
static const char *const tar_mimes[] = {
|
||||
"application/x-tar", "application/x-ustar", "application/x-gtar", NULL};
|
||||
static const char *const ogg_mimes[] = {"application/ogg", "audio/ogg",
|
||||
"video/ogg", "audio/opus", NULL};
|
||||
static const char *const ebml_mimes[] = {"video/webm", "audio/webm", NULL};
|
||||
/* ISO-BMFF, any 'ftyp' brand: containers overlap too much to split */
|
||||
static const char *const bmff_mimes[] = {"video/mp4", "audio/mp4",
|
||||
"video/quicktime", NULL};
|
||||
static const char *const avif_mimes[] = {"image/avif", NULL};
|
||||
static const char *const heic_mimes[] = {"image/heic", NULL};
|
||||
static const char *const asf_mimes[] = {"video/x-ms-asf", "video/x-ms-wmv",
|
||||
"video/x-la-asf", NULL};
|
||||
static const char *const xml_mimes[] = {"application/xml", "text/xml",
|
||||
"image/svg+xml", "image/svg-xml", NULL};
|
||||
static const char *const svg_mimes[] = {"image/svg+xml", "image/svg-xml", NULL};
|
||||
static const char *const html_mimes[] = {"text/html", NULL};
|
||||
static const char *const pem_mimes[] = {
|
||||
"application/x-x509-ca-cert", "application/x-pkcs7-certificates",
|
||||
"application/x-pkcs7-mime", "application/x-pkcs7-signature", NULL};
|
||||
|
||||
static hts_boolean mime_in(const char *const *list, const char *mime) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; list[i] != NULL; i++)
|
||||
if (strfield2(list[i], mime))
|
||||
return HTS_TRUE;
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
static hts_boolean mime_in_prefix(const char *const *list, const char *mime) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; list[i] != NULL; i++)
|
||||
if (strfield(mime, list[i]))
|
||||
return HTS_TRUE;
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
static hts_boolean has_bytes(const unsigned char *d, size_t n, size_t off,
|
||||
const char *bytes, size_t len) {
|
||||
/* overflow-safe: untrusted n alone on one side */
|
||||
return n >= off && len <= n - off && memcmp(d + off, bytes, len) == 0
|
||||
? HTS_TRUE
|
||||
: HTS_FALSE;
|
||||
}
|
||||
|
||||
static unsigned char ascii_lower(unsigned char c) {
|
||||
return c >= 'A' && c <= 'Z' ? (unsigned char) (c + 32) : c;
|
||||
}
|
||||
|
||||
/* Case-insensitive text prefix after an optional UTF-8 BOM and whitespace. */
|
||||
static hts_boolean has_text_prefix(const unsigned char *d, size_t n,
|
||||
const char *prefix) {
|
||||
const size_t len = strlen(prefix);
|
||||
size_t i, k;
|
||||
|
||||
i = n >= 3 && memcmp(d, "\xef\xbb\xbf", 3) == 0 ? 3 : 0;
|
||||
while (i < n && (d[i] == ' ' || d[i] == '\t' || d[i] == '\r' || d[i] == '\n'))
|
||||
i++;
|
||||
if (len > n - i) /* i <= n from the loop above */
|
||||
return HTS_FALSE;
|
||||
for (k = 0; k < len; k++)
|
||||
if (ascii_lower(d[i + k]) != ascii_lower((unsigned char) prefix[k]))
|
||||
return HTS_FALSE;
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
typedef enum sniff_op {
|
||||
SNIFF_QUERY_KNOWN, /* is any rule defined for this MIME? */
|
||||
SNIFF_QUERY_MATCH /* do the bytes confirm this MIME? */
|
||||
} sniff_op;
|
||||
|
||||
/* Single walk for both queries so the rule set can't drift apart. */
|
||||
static hts_boolean sniff_eval(sniff_op op, const unsigned char *d, size_t n,
|
||||
const char *mime) {
|
||||
size_t i;
|
||||
|
||||
/* KNOWN short-circuits; MATCH tests the magic */
|
||||
#define SNIFF_RULE(cond) \
|
||||
do { \
|
||||
if (op == SNIFF_QUERY_KNOWN) \
|
||||
return HTS_TRUE; \
|
||||
if (cond) \
|
||||
return HTS_TRUE; \
|
||||
} while (0)
|
||||
|
||||
for (i = 0; sniff_table[i].mime != NULL; i++) {
|
||||
if (strfield2(sniff_table[i].mime, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, sniff_table[i].off, sniff_table[i].bytes,
|
||||
sniff_table[i].len));
|
||||
}
|
||||
}
|
||||
if (mime_in(zip_mimes, mime) || mime_in_prefix(zip_mime_prefixes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "PK\3\4", 4) ||
|
||||
has_bytes(d, n, 0, "PK\5\6", 4));
|
||||
}
|
||||
if (mime_in(ole_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", 8));
|
||||
}
|
||||
if (mime_in(tar_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 257, "ustar", 5));
|
||||
}
|
||||
if (mime_in(ogg_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "OggS\0", 5));
|
||||
}
|
||||
if (mime_in(ebml_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "\x1a\x45\xdf\xa3", 4));
|
||||
}
|
||||
if (mime_in(bmff_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 4, "ftyp", 4));
|
||||
}
|
||||
if (mime_in(avif_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 4, "ftypavif", 8) ||
|
||||
has_bytes(d, n, 4, "ftypavis", 8));
|
||||
}
|
||||
if (mime_in(heic_mimes, mime)) {
|
||||
SNIFF_RULE(
|
||||
has_bytes(d, n, 4, "ftyphei", 7) || has_bytes(d, n, 4, "ftyphev", 7) ||
|
||||
has_bytes(d, n, 4, "ftypmif1", 8) || has_bytes(d, n, 4, "ftypmsf1", 8));
|
||||
}
|
||||
if (mime_in(asf_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "\x30\x26\xb2\x75\x8e\x66\xcf\x11", 8));
|
||||
}
|
||||
if (strfield2("audio/x-wav", mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "WAVE", 4));
|
||||
}
|
||||
if (strfield2("video/x-msvideo", mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "AVI ", 4));
|
||||
}
|
||||
if (strfield2("image/webp", mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) &&
|
||||
has_bytes(d, n, 8, "WEBPVP", 6));
|
||||
}
|
||||
if (strfield2("image/x-portable-anymap", mime)) {
|
||||
SNIFF_RULE(n >= 2 && d[0] == 'P' && d[1] >= '1' && d[1] <= '6');
|
||||
}
|
||||
if (strfield2("audio/x-aiff", mime)) {
|
||||
SNIFF_RULE(
|
||||
has_bytes(d, n, 0, "FORM", 4) &&
|
||||
(has_bytes(d, n, 8, "AIFF", 4) || has_bytes(d, n, 8, "AIFC", 4)));
|
||||
}
|
||||
if (strfield2("audio/mpeg", mime)) {
|
||||
/* MPEG audio frame sync (11 bits), valid layer and bitrate fields */
|
||||
SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xe0) == 0xe0 &&
|
||||
(d[1] & 0x06) != 0);
|
||||
}
|
||||
if (strfield2("audio/aac", mime)) {
|
||||
/* ADTS sync */
|
||||
SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xf6) == 0xf0);
|
||||
}
|
||||
if (strfield2("video/mp2t", mime)) {
|
||||
SNIFF_RULE(n >= 1 && d[0] == 0x47 && (n <= 188 || d[188] == 0x47));
|
||||
}
|
||||
if (mime_in(xml_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "<?xml"));
|
||||
}
|
||||
if (mime_in(svg_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "<svg") ||
|
||||
has_text_prefix(d, n, "<!DOCTYPE svg"));
|
||||
}
|
||||
if (mime_in(html_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "<!DOCTYPE") ||
|
||||
has_text_prefix(d, n, "<html") ||
|
||||
has_text_prefix(d, n, "<head"));
|
||||
}
|
||||
if (mime_in(pem_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "-----BEGIN"));
|
||||
}
|
||||
if (strfield2("audio/x-mpegurl", mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "#EXTM3U"));
|
||||
}
|
||||
if (strfield2("text/x-vcard", mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "BEGIN:VCARD"));
|
||||
}
|
||||
#undef SNIFF_RULE
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
hts_boolean hts_sniff_mime_known(const char *mime) {
|
||||
if (mime == NULL || *mime == '\0')
|
||||
return HTS_FALSE;
|
||||
return sniff_eval(SNIFF_QUERY_KNOWN, NULL, 0, mime);
|
||||
}
|
||||
|
||||
hts_boolean hts_sniff_mime_consistent(const void *data, size_t size,
|
||||
const char *mime) {
|
||||
if (data == NULL || size == 0 || mime == NULL || *mime == '\0')
|
||||
return HTS_FALSE;
|
||||
return sniff_eval(SNIFF_QUERY_MATCH, (const unsigned char *) data, size,
|
||||
mime);
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2017 Xavier Roche and other contributors
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Important notes:
|
||||
|
||||
- We hereby ask people using this source NOT to use it in purpose of grabbing
|
||||
emails addresses, or collecting any other private information on persons.
|
||||
This would disgrace our work, and spoil the many hours we spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: MIME magic-byte consistency checks */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSSNIFF_DEFH
|
||||
#define HTSSNIFF_DEFH
|
||||
|
||||
#include <stddef.h>
|
||||
#include "htsglobal.h"
|
||||
|
||||
/* Leading-body window read to arbitrate a wire/extension MIME conflict. */
|
||||
#define HTS_SNIFF_LEN 512
|
||||
|
||||
/* Can a magic rule ever confirm this MIME? (whether sniffing is worth it) */
|
||||
hts_boolean hts_sniff_mime_known(const char *mime);
|
||||
|
||||
/* TRUE when the leading body bytes are consistent with the claimed MIME;
|
||||
FALSE on unknown MIME, unknown magic, or too-short data (fail-safe). */
|
||||
hts_boolean hts_sniff_mime_consistent(const void *data, size_t size,
|
||||
const char *mime);
|
||||
|
||||
#endif
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# HT_ADD_HTMLESCAPED* must reserve the escaper's worst case (6 for _full).
|
||||
httrack -O /dev/null -#test=escape-room run | grep -q "escape-room self-test OK"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# get_ftp_line bounds a hostile CRLF-less FTP reply into its 1024-byte buffer.
|
||||
httrack -O /dev/null -#test=ftp-line run | grep -q "ftp-line self-test OK"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ftp_split_userpass bounds an over-long user:pass@ from a hostile ftp:// URL.
|
||||
httrack -O /dev/null -#test=ftp-userpass run | grep -q "ftp-userpass self-test OK"
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Response header-line parsing (treathead via -#test=header <raw-line> ...).
|
||||
# Isolates the wire layer from url_savename, which strips traversal on its own.
|
||||
|
||||
hdr() {
|
||||
local want="$1"
|
||||
shift
|
||||
out="$(httrack -O /dev/null -#test=header "$@" | grep '^contenttype=')"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: $* -> '$out' (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
hdr 'contenttype=application/pdf cdispo=' 'Content-Type: application/pdf'
|
||||
|
||||
# filename= is honored quoted or bare.
|
||||
hdr 'contenttype= cdispo=report.pdf' \
|
||||
'Content-Disposition: attachment; filename="report.pdf"'
|
||||
hdr 'contenttype= cdispo=report.pdf' \
|
||||
'Content-Disposition: attachment; filename=report.pdf'
|
||||
|
||||
# Path components in the filename are dropped on the wire (RFC 2616).
|
||||
hdr 'contenttype= cdispo=evil.pdf' \
|
||||
'Content-Disposition: attachment; filename="../../evil.pdf"'
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# inplace_escape_*() must match escape_*() on a copy: guards the shared helper.
|
||||
httrack -O /dev/null -#test=inplace-escape run | grep -q "inplace-escape self-test OK"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# .class constant-pool count is capped to the file size (calloc DoS).
|
||||
httrack -O /dev/null -#test=java run | grep -q "java constant-pool cap self-test OK"
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# hts_finish_makeindex writes the footer and gates the refresh meta on a single
|
||||
# first link (guards the macro->function extraction).
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=makeindex "$dir" run |
|
||||
grep -q "makeindex self-test OK"
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Cache generation reconcile policies (httrack -#test=reconcile <dir>):
|
||||
# promote a stranded old generation, keep the larger one after an aborted
|
||||
# run, and restore the old one when an update transferred nothing.
|
||||
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
out=$(httrack -#test=reconcile "$dir")
|
||||
|
||||
test "$out" = "cache-reconcile: OK" || {
|
||||
echo "expected 'cache-reconcile: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# #159: a redirect to a same-file alias (http<->https, user@host, ..) must be
|
||||
# followed through, not turned into a self-pointing "moved" stub. The decision
|
||||
# helper is exercised by the engine self-test.
|
||||
httrack -O /dev/null -#test=redirect-samefile run | grep -q "redirect-samefile self-test OK"
|
||||
@@ -3,38 +3,13 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Local save-name resolution (url_savename via -#test=savename <fil> <content-type> [key=value ...]).
|
||||
# name() asserts on the basename, full() on the whole path; prior= registers an
|
||||
# already-crawled link whose sav is rooted under the -O path (/dev/null here).
|
||||
|
||||
# resolve httrack before cd: make check puts a RELATIVE ../src on PATH
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
# scratch dir: body= and cached= write temp files (st-savename-body.tmp, hts-cache/)
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
run() {
|
||||
"$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
|
||||
}
|
||||
# Local save-name extension resolution (url_savename via -#test=savename <fil> <content-type>).
|
||||
# Asserts on the basename of "savename: <path>".
|
||||
|
||||
name() {
|
||||
local fil="$1" ctype="$2" want="$3"
|
||||
shift 3
|
||||
out="$(run "$fil" "$ctype" "$@")"
|
||||
test "${out##*/}" == "$want" || {
|
||||
echo "FAIL: '$fil' '$ctype' $* -> '$out' (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
full() {
|
||||
local fil="$1" ctype="$2" want="$3"
|
||||
shift 3
|
||||
out="$(run "$fil" "$ctype" "$@")"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: '$fil' '$ctype' $* -> '$out' (want '$want')"
|
||||
out="$(httrack -O /dev/null -#test=savename "$1" "$2" | sed -n 's/^savename: //p')"
|
||||
test "${out##*/}" == "$3" || {
|
||||
echo "FAIL: '$1' '$2' -> '$out' (want '$3')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
@@ -64,95 +39,3 @@ name '/types/data.json' 'application/json' 'data.json'
|
||||
|
||||
# Agreeing type must not rewrite the extension's casing (no strip-and-reappend).
|
||||
name '/x.JPG' 'image/jpeg' 'x.JPG'
|
||||
|
||||
# A Content-Disposition filename replaces the URL name outright.
|
||||
name '/x.php' 'application/pdf' 'report.pdf' cdispo=report.pdf
|
||||
name '/download' 'text/html' 'setup.exe' cdispo=setup.exe
|
||||
|
||||
# Reserved characters in a hostile Content-Disposition name are sanitized.
|
||||
name '/x.php' 'application/pdf' 'set_up.exe' 'cdispo=set:up.exe'
|
||||
|
||||
# The md5-of-query suffix lands inside a Content-Disposition name too.
|
||||
name '/x.php?id=1' 'application/pdf' 'report681a.pdf' cdispo=report.pdf
|
||||
|
||||
# Still-downloading path (status=-1): mime drives the ext, cdispo is ignored
|
||||
# there (the deliberately unfolded 4th resolve_extension variant).
|
||||
name '/x.pdf' 'text/html' 'x.html' status=-1
|
||||
name '/x.html' 'text/html' 'x.html' status=-1
|
||||
name '/x.php' 'application/pdf' 'x.pdf' status=-1 cdispo=report.pdf
|
||||
|
||||
# Contested type (wire disagrees with a specific ext): magic bytes proving the
|
||||
# extension right keep it, anything else trusts the wire as before.
|
||||
name '/photo.jpg' 'image/png' 'photo.jpg' body=hex:FFD8FFE000104A46
|
||||
name '/photo.jpg' 'image/png' 'photo.png' body=hex:89504E470D0A1A0A
|
||||
name '/photo.jpg' 'image/png' 'photo.png'
|
||||
name '/doc.pdf' 'text/html' 'doc.pdf' body=hex:255044462D312E34
|
||||
name '/doc.pdf' 'text/html' 'doc.html' 'body=<html><body>soft 404</body></html>'
|
||||
name '/style.css' 'image/png' 'style.png' 'body=body { }' # no rule for css: wire wins
|
||||
|
||||
# A redirect answer resolves nothing: delayed placeholder name.
|
||||
name '/x.php' 'text/html' 'x.0.delayed' statuscode=301
|
||||
|
||||
# Root and query-only URLs get index + the md5-of-query suffix.
|
||||
name '/' 'text/html' 'index.html'
|
||||
name '/?a=1' 'text/html' 'index3872.html'
|
||||
|
||||
# Same URL crawled before: reuse its sav verbatim (case preserved).
|
||||
full '/X.PHP' 'text/html' 'www.example.com/CASE.HTML' \
|
||||
'prior=www.example.com|/X.PHP|www.example.com/CASE.HTML'
|
||||
|
||||
# Another URL owns the name: collision suffix -2, then -3, case-insensitively.
|
||||
name '/x.php' 'text/html' 'x-2.html' \
|
||||
'prior=www.example.com|/other.html|/dev/null/www.example.com/x.html'
|
||||
name '/x.php' 'text/html' 'x-3.html' \
|
||||
'prior=www.example.com|/o1.html|/dev/null/www.example.com/x.html' \
|
||||
'prior=www.example.com|/o2.html|/dev/null/www.example.com/x-2.html'
|
||||
name '/INDEX.HTML' 'text/html' 'INDEX-2.HTML' \
|
||||
'prior=www.example.com|/index.html|/dev/null/www.example.com/index.html'
|
||||
|
||||
# Same basename in another directory is NOT a collision.
|
||||
name '/x.php' 'text/html' 'x.html' \
|
||||
'prior=www.example.com|/sub/x.html|/dev/null/www.example.com/sub/x.html'
|
||||
|
||||
# 8-3 modes: DOS truncates every component to 8+3, ISO9660 level 2 to 31.
|
||||
full '/directory-long/verylongfilename.html' 'text/html' \
|
||||
'/dev/null/EXAMPLE/DIRECTOR/VERYLONG.HTM' n83=1
|
||||
full '/directory-long/verylongfilename.html' 'text/html' \
|
||||
'/dev/null/EXAMPLE_C/DIRECTORY_LONG/VERYLONGFILENAME.HTM' n83=2
|
||||
name '/verylongfilename.php' 'text/html' 'VERYLO-2.HTM' n83=1 \
|
||||
'prior=www.example.com|/other.html|/dev/null/EXAMPLE/VERYLONG.HTM'
|
||||
|
||||
# urlhack dedup (#271): // collapse and www-strip map to the prior link's sav;
|
||||
# the per-feature negatives opt out and take a fresh name.
|
||||
full '/a//b.php' 'text/html' '/dev/null/www.example.com/a/PRIOR.html' \
|
||||
'prior=www.example.com|/a/b.php|/dev/null/www.example.com/a/PRIOR.html'
|
||||
full '/a//b.php' 'text/html' '/dev/null/www.example.com/a/b.html' no-slash=1 \
|
||||
'prior=www.example.com|/a/b.php|/dev/null/www.example.com/a/PRIOR.html'
|
||||
full '/w.php' 'text/html' '/dev/null/www.example.com/W-PRIOR.html' adr=example.com \
|
||||
'prior=www.example.com|/w.php|/dev/null/www.example.com/W-PRIOR.html'
|
||||
full '/w.php' 'text/html' '/dev/null/example.com/w.html' adr=example.com no-www=1 \
|
||||
'prior=www.example.com|/w.php|/dev/null/www.example.com/W-PRIOR.html'
|
||||
|
||||
# Distinct URLs must stay distinct under urlhack (no over-normalization).
|
||||
full '/a//b.php' 'text/html' '/dev/null/www.example.com/a/b.html' \
|
||||
'prior=www.example.com|/a/c.php|/dev/null/www.example.com/a/C-PRIOR.html'
|
||||
|
||||
# --strip-query (#112): stripped key dedups onto the prior sav; without the
|
||||
# option the same URLs stay distinct.
|
||||
full '/page.php?id=3&sid=42' 'text/html' '/dev/null/www.example.com/PAGE-PRIOR.html' \
|
||||
strip=sid 'prior=www.example.com|/page.php?id=3|/dev/null/www.example.com/PAGE-PRIOR.html'
|
||||
full '/page.php?id=3&sid=42' 'text/html' '/dev/null/www.example.com/page475b.html' \
|
||||
'prior=www.example.com|/page.php?id=3|/dev/null/www.example.com/PAGE-PRIOR.html'
|
||||
|
||||
# A kept key that differs must still block the dedup (no over-stripping).
|
||||
full '/page.php?id=3&sid=42' 'text/html' '/dev/null/www.example.com/page475b.html' \
|
||||
strip=sid 'prior=www.example.com|/page.php?id=4|/dev/null/www.example.com/PAGE-PRIOR.html'
|
||||
|
||||
# Hostile fils stay rooted under the mirror: ../ (raw or %2e-encoded) drops out,
|
||||
# control characters become spaces, oversized names cap at 210 chars (the cap
|
||||
# can chop the extension off entirely).
|
||||
full '/../../etc/passwd' 'text/html' '/dev/null/www.example.com///etc/passwd.html'
|
||||
full '/%2e%2e/%2e%2e/etc/passwd' 'text/html' '/dev/null/www.example.com///etc/passwd.html'
|
||||
full '/x.php' 'application/pdf' '/dev/null/www.example.com///evil.exe' 'cdispo=../../evil.exe'
|
||||
name $'/evil\rname\t.php' 'text/html' 'evil name .html'
|
||||
name "/$(printf 'a%.0s' {1..300}).php" 'text/html' "$(printf 'a%.0s' {1..210})"
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# MIME magic consistency (-#test=sniff <content-type> <hex:..|text>), the
|
||||
# tie-break behind htsname's wire-vs-extension naming.
|
||||
|
||||
chk() {
|
||||
local mime="$1" body="$2" want="$3"
|
||||
out="$(httrack -#test=sniff "$mime" "$body" | sed -n 's/^sniff: //p')"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: '$mime' '$body' -> '$out' (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
yes='known=1 consistent=1'
|
||||
no='known=1 consistent=0'
|
||||
unk='known=0 consistent=0'
|
||||
|
||||
# images
|
||||
chk image/jpeg hex:FFD8FFE000104A46 "$yes"
|
||||
chk image/png hex:89504E470D0A1A0A "$yes"
|
||||
chk image/png hex:FFD8FFE000104A46 "$no" # jpeg bytes are not a png
|
||||
chk image/gif 'GIF89a' "$yes"
|
||||
chk image/bmp 'BMxxxx' "$yes"
|
||||
chk image/tiff hex:49492A00 "$yes"
|
||||
chk image/tiff hex:4D4D002A "$yes" # both endians
|
||||
chk image/x-icon hex:00000100 "$yes"
|
||||
chk image/x-icon hex:00000200 "$yes" # Windows cursor, spec maps to x-icon
|
||||
chk image/webp 'RIFFxxxxWEBPVP' "$yes"
|
||||
chk image/webp 'RIFFxxxxWAVE' "$no" # riff subtype discriminates
|
||||
chk image/avif hex:0000001C6674797061766966 "$yes"
|
||||
chk image/avif hex:0000001C6674797068656963 "$no" # heic brand is not avif
|
||||
chk image/heic hex:0000001C6674797068656963 "$yes"
|
||||
chk image/svg+xml '<svg xmlns="x">' "$yes"
|
||||
chk image/svg+xml $'\xef\xbb\xbf <?xml version="1.0"?>' "$yes" # BOM+ws skip
|
||||
|
||||
# audio / video
|
||||
chk audio/mpeg 'ID3xxx' "$yes"
|
||||
chk audio/mpeg hex:FFFB9000 "$yes" # bare frame sync
|
||||
chk audio/aac hex:FFF15080 "$yes"
|
||||
chk audio/flac 'fLaC' "$yes"
|
||||
chk audio/ogg hex:4F67675300 "$yes"
|
||||
chk audio/x-wav 'RIFFxxxxWAVE' "$yes"
|
||||
chk video/x-msvideo 'RIFFxxxxAVI ' "$yes"
|
||||
chk video/x-msvideo 'RIFFxxxxWAVE' "$no"
|
||||
chk video/mp4 hex:000000186674797069736F6D "$yes"
|
||||
chk video/webm hex:1A45DFA3 "$yes"
|
||||
chk video/mpeg hex:000001BA "$yes"
|
||||
chk video/x-ms-wmv hex:3026B2758E66CF11 "$yes"
|
||||
|
||||
# archives; zip magic covers the office-container families
|
||||
chk application/zip hex:504B0304 "$yes"
|
||||
chk application/vnd.openxmlformats-officedocument.wordprocessingml.document hex:504B0304 "$yes"
|
||||
chk application/vnd.oasis.opendocument.text hex:504B0304 "$yes"
|
||||
chk application/msword hex:D0CF11E0A1B11AE1 "$yes"
|
||||
chk application/msword hex:504B0304 "$no" # legacy .doc is OLE, not zip
|
||||
chk application/x-gzip hex:1F8B08 "$yes"
|
||||
chk application/x-bzip2 'BZh9' "$yes"
|
||||
chk application/x-7z-compressed hex:377ABCAF271C "$yes"
|
||||
chk application/x-rar-compressed hex:526172211A07 "$yes"
|
||||
chk application/zstd hex:28B52FFD "$yes"
|
||||
chk application/x-tar "hex:$(printf '00%.0s' {1..257})7573746172" "$yes" # ustar at 257
|
||||
chk application/x-tar hex:7573746172 "$no"
|
||||
|
||||
# documents, fonts, misc
|
||||
chk application/pdf '%PDF-1.7' "$yes"
|
||||
chk application/pdf '<html><body>soft 404</body></html>' "$no"
|
||||
chk application/postscript '%!PS-Adobe' "$yes"
|
||||
chk application/rtf '{\rtf1' "$yes"
|
||||
chk font/woff2 'wOF2' "$yes"
|
||||
chk font/otf 'OTTO' "$yes"
|
||||
chk font/ttf hex:0001000000 "$yes"
|
||||
chk application/x-shockwave-flash 'CWSx' "$yes"
|
||||
chk application/x-java-vm hex:CAFEBABE "$yes"
|
||||
chk application/wasm hex:0061736D "$yes"
|
||||
chk text/html $' \r\n<!DOCTYPE html><html>' "$yes"
|
||||
chk text/html '<html lang="en">' "$yes"
|
||||
chk text/html 'plain text, no markup' "$no"
|
||||
chk text/xml '<?xml version="1.0"?>' "$yes"
|
||||
|
||||
# no magic rule at all: never confirmed, never blocks the wire type
|
||||
chk text/css 'body { }' "$unk"
|
||||
chk text/plain 'hello' "$unk"
|
||||
chk application/x-javascript 'var x;' "$unk"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Entity/URL unescapers reserve one byte for the trailing NUL (no 1-byte OOB).
|
||||
httrack -O /dev/null -#test=unescape-bounds run | grep -q "unescape-bounds self-test OK"
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Read-side cache corruption (httrack -#test=cache-corrupt <dir>): zip byte
|
||||
# surgery (bad/oversized X-Size, blanked X-In-Cache, smashed header, garbled
|
||||
# deflate) must each be rejected per-entry, never crash, never taint the sibling.
|
||||
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# the smashed-header case logs expected "Corrupted cache entry" warnings on
|
||||
# stdout; the verdict is the last line
|
||||
out=$(httrack -#test=cache-corrupt "$dir" 2>/dev/null | tail -n1)
|
||||
|
||||
test "$out" = "cache-corrupt: OK" || {
|
||||
echo "expected 'cache-corrupt: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -4,9 +4,10 @@
|
||||
# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
|
||||
# tool flags despite the #!/bin/bash above.
|
||||
|
||||
# Cache write-failure policy (-#test=cache-writefail <dir>). #174/#219: disk
|
||||
# full or a failure streak aborts cleanly; an isolated failure or an oversized
|
||||
# entry is only dropped.
|
||||
# Cache write-failure handling (httrack -#test=cache-writefail <dir>). #174/#219.
|
||||
# A failing new.zip write (disk full) used to crash the process via assertf; it
|
||||
# must instead stop the mirror with a fatal error (exit_xh=-1), no crash. The
|
||||
# self-test asserts that; reverting the fix makes -#test=cache-writefail abort (SIGABRT) and fail.
|
||||
|
||||
set -eu
|
||||
|
||||
@@ -21,9 +22,3 @@ printf '%s\n' "$out" | grep -qx "cache-writefail: OK" || {
|
||||
echo "expected 'cache-writefail: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# A skipped entry must be warned about with its URL.
|
||||
printf '%s\n' "$out" | grep -q "entry not cached: example.com/" || {
|
||||
echo "expected a URL-bearing skip warning" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Update-run naming from a real cache entry (-#test=savename cached=<ctype>|<save>).
|
||||
# Named 01_zlib-*: the cache writer needs zlib, which the MSan job can't run.
|
||||
|
||||
# resolve httrack before cd: make check puts a RELATIVE ../src on PATH
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
name() {
|
||||
local fil="$1" ctype="$2" want="$3"
|
||||
shift 3
|
||||
out="$("$httrack_bin" -O /dev/null -#test=savename "$fil" "$ctype" "$@" | sed -n 's/^savename: //p')"
|
||||
test "${out##*/}" == "$want" || {
|
||||
echo "FAIL: '$fil' '$ctype' $* -> '$out' (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# No live bytes: the recorded save name (X-Save) reproduces the previous
|
||||
# verdict; cached body bytes (PNG magic) are ignored; css has no magic rule.
|
||||
name '/photo.jpg' 'image/png' 'photo.jpg' 'cached=image/png|www.example.com/photo.jpg'
|
||||
name '/photo.jpg' 'image/png' 'photo.png' 'cached=image/png|www.example.com/photo.png'
|
||||
name '/photo.jpg' 'image/jpeg' 'photo.jpg' 'cached=image/jpeg|www.example.com/photo.png'
|
||||
name '/style.css' 'image/png' 'style.css' 'cached=image/png|www.example.com/style.css'
|
||||
# agreement keeps the URL ext verbatim (.jpeg), never canonicalized to .jpg
|
||||
name '/photo.jpeg' 'image/jpeg' 'photo.jpeg' 'cached=image/jpeg|www.example.com/photo.jpeg'
|
||||
@@ -1,10 +1,11 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Content-Type vs URL-extension naming (#267 family, default -%N2). A MISSING
|
||||
# type keeps a specific non-HTML ext; a DECLARED disagreeing type is trusted
|
||||
# unless magic bytes prove the ext right (lie/wrongtype/packed keep theirs),
|
||||
# so a real HTML body (report.pdf) still becomes .html. Wrong names are
|
||||
# asserted absent so a regression in either direction fails.
|
||||
# Content-Type vs URL-extension naming (issue #267 family) under the default
|
||||
# delayed type check (-%N2). Policy: a MISSING Content-Type must not clobber a
|
||||
# URL extension that maps to a specific non-HTML type (.png/.pdf stay as-is);
|
||||
# an explicitly DECLARED type is trusted, so a binary-looking URL that really
|
||||
# serves HTML (text/html on .pdf/.jpg) is named .html. The "wrong" names are
|
||||
# asserted absent so a regression in either direction fails here.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
@@ -13,11 +14,7 @@ bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'types/notype.pdf' --not-found 'types/notype.html' \
|
||||
--found 'types/photo.png' \
|
||||
--found 'types/doc.pdf' \
|
||||
--found 'types/lie.png' --not-found 'types/lie.html' \
|
||||
--found 'types/wrongtype.jpg' --not-found 'types/wrongtype.png' \
|
||||
--found 'types/bigtype.jpg' --not-found 'types/bigtype.png' \
|
||||
--found 'types/mutant.jpg' --not-found 'types/mutant.png' \
|
||||
--found 'types/packed.jpg' --not-found 'types/packed.png' \
|
||||
--found 'types/lie.html' --not-found 'types/lie.png' \
|
||||
--found 'types/report.html' --not-found 'types/report.pdf' \
|
||||
--found 'types/page.htm' --not-found 'types/page.html' \
|
||||
--found 'types/script.js' \
|
||||
|
||||
@@ -1,18 +1,15 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An update pass keeps the names the first crawl chose: type and save name
|
||||
# ride the cache, so a declared-text/html .pdf stays .html, a typeless .png
|
||||
# stays .png, and a sniff-kept ext is reproduced from X-Save even when the
|
||||
# refetched content changed (mutant.jpg serves PNG bytes on the rerun).
|
||||
# A second (update) pass must keep the names the first crawl chose. The stored
|
||||
# Content-Type rides the cache, so the update reads back the same value -- the
|
||||
# unknown/unknown sentinel for a typeless response, the declared type otherwise
|
||||
# -- and names consistently: a declared-text/html .pdf stays .html and a
|
||||
# typeless .png stays .png across the update rather than reverting.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
|
||||
--found 'types/report.html' --not-found 'types/report.pdf' \
|
||||
--found 'types/notype.png' --not-found 'types/notype.html' \
|
||||
--found 'types/lie.png' --not-found 'types/lie.html' \
|
||||
--found 'types/wrongtype.jpg' --not-found 'types/wrongtype.png' \
|
||||
--found 'types/bigtype.jpg' --not-found 'types/bigtype.png' \
|
||||
--found 'types/packed.jpg' --not-found 'types/packed.png' \
|
||||
--found 'types/mutant.jpg' --not-found 'types/mutant.png' \
|
||||
--found 'types/lie.html' \
|
||||
httrack 'BASEURL/types/index.html'
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
#!/bin/bash
|
||||
# Issues #32/#41: a Content-Length that disagrees with the body warns
|
||||
# "incomplete transfer" and skips the cache; -%B (tolerant) accepts it.
|
||||
|
||||
set -euo pipefail
|
||||
# Issues #32/#41: a Content-Length that disagrees with the body warns "bogus
|
||||
# state (broken size)" and skips the cache; -%B (tolerant) accepts it.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# Default: warn, but the file is still written.
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'size/oversize.bin' \
|
||||
--log-found 'incomplete transfer \(expected' \
|
||||
--log-found 'bogus state \(broken size' \
|
||||
httrack 'BASEURL/size/index.html'
|
||||
|
||||
# -%B (tolerant): no warning, file written.
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'size/oversize.bin' \
|
||||
--log-not-found 'incomplete transfer|not cached' \
|
||||
--log-not-found 'bogus state' \
|
||||
httrack 'BASEURL/size/index.html' '-%B'
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Issue #279: an anchored link (target.html#sec, quoted or bare) fetches the
|
||||
# target with the fragment dropped (strict server 400s on a '#' in the request)
|
||||
# but keeps it in the rewritten local link so the anchor still works.
|
||||
set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'fraglink/target.html' \
|
||||
--file-matches 'fraglink/index.html' 'href=target\.html#sec' \
|
||||
--file-matches 'fraglink/index.html' 'href="target\.html#sec2"' \
|
||||
httrack 'BASEURL/fraglink/index.html'
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/bin/bash
|
||||
# The java plugin must load (versioned dlopen name) and parse a .class
|
||||
# constant pool: a resource named only inside Foo.class gets crawled.
|
||||
set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
tmproot=$(mktemp -d)
|
||||
trap 'rm -rf "$tmproot"' EXIT
|
||||
mkdir "$tmproot/javaclass"
|
||||
|
||||
cat >"$tmproot/javaclass/index.html" <<'EOF'
|
||||
<html><body><a href="Foo.class">applet</a></body></html>
|
||||
EOF
|
||||
printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
|
||||
# magic/minor/major, count=2, one CONSTANT_Utf8 "hello.gif", class/superclass
|
||||
printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
|
||||
>"$tmproot/javaclass/Foo.class"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
|
||||
--found 'javaclass/Foo.class' \
|
||||
--found 'javaclass/hello.gif' \
|
||||
httrack 'BASEURL/javaclass/index.html'
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Content-Disposition names the saved file: the attachment filename replaces
|
||||
# the URL-derived name, and a traversal filename is reduced to its last
|
||||
# component, inside the mirror.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'cdispo/report.pdf' \
|
||||
--file-matches 'cdispo/report.pdf' '%PDF' \
|
||||
--not-found 'cdispo/fetch.pdf' \
|
||||
--found 'cdispo/evil.pdf' \
|
||||
--not-found 'evil.pdf' \
|
||||
httrack 'BASEURL/cdispo/index.html'
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Degenerate delayed-type paths (#5/#107 family): redirects that never resolve
|
||||
# a name must drop cleanly -- no .delayed leftovers (audited by local-crawl.sh),
|
||||
# no "not cached" warnings, resolvable links still land correctly.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --rerun --errors 0 \
|
||||
--found 'delayed/real.pdf' \
|
||||
--file-matches 'delayed/real.pdf' '%PDF' \
|
||||
--found 'delayed/notype.bin.html' \
|
||||
--found 'delayed/empty.html' \
|
||||
--not-found 'delayed/noloc.html' \
|
||||
--not-found 'delayed/selfloop.html' \
|
||||
--not-found 'delayed/chain9.pdf' \
|
||||
--log-not-found 'not cached' \
|
||||
httrack 'BASEURL/delayed/index.html'
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# -E time limit (#481): server pages trickle for minutes; the engine must stop
|
||||
# on its own at -E plus grace, aborting the in-flight transfers.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
|
||||
start=$(date +%s)
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--skip-delayed-audit \
|
||||
--log-found 'More than 2 seconds passed' \
|
||||
httrack 'BASEURL/trickle/index.html' -E2 -c4
|
||||
wall=$(($(date +%s) - start))
|
||||
# hard stop is due at -E2 + 5s grace; near TRICKLE_SECONDS means it never fired
|
||||
if [ "$wall" -ge 30 ]; then
|
||||
echo "crawl took ${wall}s, -E hard stop did not engage" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# -M byte cap (#77): the crawl must stop with the "giving up" error and keep
|
||||
# the mirror well under the 8 x 640KB the fixture totals uncapped.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# cap = -M + the 4 in-flight files the smooth stop lets finish + one of margin
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--log-found 'More than 400000 bytes have been transferred.. giving up' \
|
||||
--found bigfiles/p0.bin \
|
||||
--max-mirror-bytes 3700000 \
|
||||
httrack 'BASEURL/bigfiles/index.html' -M400000 -c4
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
|
||||
# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
|
||||
# family + 6 singles; the 4 planted errors write -o1 pages, not counted.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --rerun \
|
||||
--errors 4 --files 360 \
|
||||
--found 'big/p/95.html' \
|
||||
--found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
|
||||
--found 'big/a/f2-2x.png' \
|
||||
--found 'big/a/subs.vtt' \
|
||||
--found 'big/a/font.woff2' \
|
||||
--found 'big/a/js-data.bin' \
|
||||
--found 'big/d/01.pdf' \
|
||||
--found 'big/d/named.pdf' \
|
||||
--found 'big/a/doc.pdf' \
|
||||
--found "big/f9/caf$(printf '\xc3\xa9').html" \
|
||||
--found 'big/f7/fa.html' \
|
||||
--found 'big/a/ref.png' \
|
||||
--found 'big/f6/sub/leaf.html' \
|
||||
--found 'big/f1/dir/index.html' \
|
||||
--found 'big/f10/empty.html' \
|
||||
--found 'big/indexd41d.html' \
|
||||
--found 'big/a/i0a.png' \
|
||||
--not-found 'big/x/og' \
|
||||
--not-found 'big/x/tw' \
|
||||
--not-found 'big/x/jsonld.png' \
|
||||
--not-found 'big/x/never-scanned.png' \
|
||||
--not-found 'big/x/atom-only.html' \
|
||||
--not-found 'big/x/sitemap-only.html' \
|
||||
--not-found 'big/x/form-target.html' \
|
||||
--not-found 'big/x/formact' \
|
||||
--not-found 'big/x/ping' \
|
||||
--not-found 'big/x/aj.jar' \
|
||||
--not-found 'big/x/bj.jar' \
|
||||
--not-found 'big/x/is1.png' \
|
||||
--not-found 'big/x/concat.html' \
|
||||
--file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
|
||||
--file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
|
||||
--file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \
|
||||
--file-not-matches 'big/p/1.html' 'href="/big/' \
|
||||
--log-not-found 'not cached|[Pp]anic|assert' \
|
||||
--log-found '\(404\) at link [^ ]*/big/e/404\.html' \
|
||||
--log-found '\(410\) at link [^ ]*/big/e/410\.html' \
|
||||
--log-found '\(500\) at link [^ ]*/big/e/500\.html' \
|
||||
--log-found 'decompressing.*big/e/gztrunc\.html' \
|
||||
--log-found ', no files updated' \
|
||||
--max-mirror-bytes 700000 \
|
||||
--min-mirror-bytes 500000 \
|
||||
httrack 'BASEURL/big/index.html' --retries=0 -c8 -%c100 -A100000000
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An update run against a dead server must not destroy the cache: the no-data
|
||||
# rollback restores the previous hts-cache generation (zip caches lost it).
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun-dead \
|
||||
--found 'simple/basic.html' \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
@@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An all-304 update of a tiny site (headers under the 32K rollback threshold)
|
||||
# is a healthy run: it must not trip the no-data rollback as a fake outage.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
|
||||
--log-found 'no files updated' \
|
||||
--log-not-found 'No data seems to have been transferred' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html'
|
||||
@@ -6,7 +6,6 @@ EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
local-crawl.sh local-server.py server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
server-root/fraglink/index.html server-root/fraglink/target.html \
|
||||
fixtures/cache-golden/hts-cache/new.zip
|
||||
|
||||
TESTS_ENVIRONMENT =
|
||||
@@ -35,39 +34,26 @@ TESTS = \
|
||||
01_engine-entities.test \
|
||||
01_engine-filelist.test \
|
||||
01_engine-filter.test \
|
||||
01_engine-ftp-line.test \
|
||||
01_engine-ftp-userpass.test \
|
||||
01_engine-hashtable.test \
|
||||
01_engine-header.test \
|
||||
01_engine-idna.test \
|
||||
01_engine-escape-room.test \
|
||||
01_engine-inplace-escape.test \
|
||||
01_engine-java.test \
|
||||
01_engine-makeindex.test \
|
||||
01_engine-mime.test \
|
||||
01_engine-parse.test \
|
||||
01_engine-pause.test \
|
||||
01_engine-rcfile.test \
|
||||
01_engine-reconcile.test \
|
||||
01_engine-redirect.test \
|
||||
01_engine-relative.test \
|
||||
01_engine-robots.test \
|
||||
01_engine-savename.test \
|
||||
01_engine-selftest-dispatch.test \
|
||||
01_engine-simplify.test \
|
||||
01_engine-sniff.test \
|
||||
01_engine-status.test \
|
||||
01_engine-stripquery.test \
|
||||
01_engine-strsafe.test \
|
||||
01_engine-urlhack.test \
|
||||
01_engine-unescape-bounds.test \
|
||||
01_engine-useragent.test \
|
||||
01_zlib-acceptencoding.test \
|
||||
01_zlib-cache.test \
|
||||
01_zlib-cache-corrupt.test \
|
||||
01_zlib-cache-golden.test \
|
||||
01_zlib-cache-writefail.test \
|
||||
01_zlib-savename-cached.test \
|
||||
02_manpage-regen.test \
|
||||
02_update-cache.test \
|
||||
10_crawl-simple.test \
|
||||
@@ -94,15 +80,6 @@ TESTS = \
|
||||
26_local-strip-query.test \
|
||||
27_local-cookies-file.test \
|
||||
28_local-pause.test \
|
||||
29_local-redirect-fragment.test \
|
||||
30_local-fragment-link.test \
|
||||
31_local-javaclass.test \
|
||||
32_local-cdispo.test \
|
||||
33_local-delayed.test \
|
||||
34_local-maxtime.test \
|
||||
35_local-maxsize.test \
|
||||
36_local-bigcrawl.test \
|
||||
37_local-cache-outage.test \
|
||||
38_local-update-304.test
|
||||
29_local-redirect-fragment.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -15,18 +15,11 @@
|
||||
# bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
|
||||
# --errors N --files N --found PATH ... --directory PATH ... \
|
||||
# --log-found REGEX ... --log-not-found REGEX ... \
|
||||
# --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
|
||||
# --max-mirror-bytes N \
|
||||
# httrack BASEURL/some/path [httrack-args...]
|
||||
# --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
|
||||
# --max/--min-mirror-bytes bound the mirrored content bytes (host root).
|
||||
# --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
|
||||
# host root), to assert rewritten link/content survived the crawl.
|
||||
# --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
|
||||
# which the ephemeral port forces into the cookie domain) and passes it to
|
||||
# httrack via --cookies-file, to exercise preloaded cookies.
|
||||
# --rerun-dead re-runs with the server stopped: the no-data rollback must
|
||||
# restore the previous hts-cache generation byte-identical.
|
||||
|
||||
set -u
|
||||
|
||||
@@ -39,7 +32,6 @@ key="${testdir}/server.key"
|
||||
tls=
|
||||
verbose=
|
||||
rerun=
|
||||
rerun_dead=
|
||||
tmpdir=
|
||||
serverpid=
|
||||
crawlpid=
|
||||
@@ -97,7 +89,6 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
|
||||
# --- parse leading control flags --------------------------------------------
|
||||
declare -a audit=()
|
||||
declare -a cookies=()
|
||||
skip_delayed_audit=""
|
||||
scheme=http
|
||||
pos=0
|
||||
args=("$@")
|
||||
@@ -105,8 +96,7 @@ nargs=$#
|
||||
while test "$pos" -lt "$nargs"; do
|
||||
case "${args[$pos]}" in
|
||||
--debug) verbose=1 ;;
|
||||
--rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
|
||||
--rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
|
||||
--rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
|
||||
--no-purge)
|
||||
nopurge=1
|
||||
audit+=("--no-purge")
|
||||
@@ -123,21 +113,14 @@ while test "$pos" -lt "$nargs"; do
|
||||
pos=$((pos + 1))
|
||||
cookies+=("${args[$pos]}")
|
||||
;;
|
||||
--skip-delayed-audit)
|
||||
skip_delayed_audit=1
|
||||
;;
|
||||
--errors | --files)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
;;
|
||||
--found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes | --min-mirror-bytes)
|
||||
--found | --not-found | --directory | --log-found | --log-not-found)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
;;
|
||||
--file-matches | --file-not-matches)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
|
||||
pos=$((pos + 2))
|
||||
;;
|
||||
httrack)
|
||||
pos=$((pos + 1))
|
||||
break
|
||||
@@ -245,43 +228,6 @@ if test -n "$rerun"; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- optional dead pass: server stopped, the cache must survive the rollback --
|
||||
if test -n "$rerun_dead"; then
|
||||
zip="${out}/hts-cache/new.zip"
|
||||
test -s "$zip" || die "no cache was written by the first pass"
|
||||
cp "$zip" "${tmpdir}/cache-before.zip"
|
||||
cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
serverpid=
|
||||
info "re-running httrack against the stopped server"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid" || true
|
||||
crawlpid=
|
||||
result "OK (dead pass ran)"
|
||||
# The dead pass must have gone through the no-data rollback, not bailed out
|
||||
# before the mirror loop (which would leave the cache trivially untouched).
|
||||
info "checking the dead pass hit the rollback"
|
||||
if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
|
||||
result "OK"
|
||||
else
|
||||
result "rollback notice not found in hts-log.txt"
|
||||
exit 1
|
||||
fi
|
||||
info "checking the previous cache generation was restored"
|
||||
if cmp -s "$zip" "${tmpdir}/cache-before.zip" &&
|
||||
test ! -e "${out}/hts-cache/old.zip"; then
|
||||
result "OK"
|
||||
else
|
||||
result "new.zip differs from the pre-outage cache (or old.zip left behind)"
|
||||
exit 1
|
||||
fi
|
||||
# Audits below describe the healthy crawl, not the dead pass.
|
||||
cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
|
||||
fi
|
||||
|
||||
# --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
|
||||
hostroot=
|
||||
for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
|
||||
@@ -293,17 +239,6 @@ done
|
||||
test -n "$hostroot" || die "could not find host root under $out"
|
||||
debug "host root: $hostroot"
|
||||
|
||||
# A completed crawl must leave no .delayed temporaries (issue #107).
|
||||
# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
|
||||
if test -z "$skip_delayed_audit"; then
|
||||
info "checking for leftover .delayed files"
|
||||
leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
|
||||
if test -z "$leftovers"; then result "OK"; else
|
||||
result "leftover: $leftovers"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- audit -------------------------------------------------------------------
|
||||
i=0
|
||||
while test "$i" -lt "${#audit[@]}"; do
|
||||
@@ -359,42 +294,6 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
exit 1
|
||||
else result "OK"; fi
|
||||
;;
|
||||
--max-mirror-bytes)
|
||||
i=$((i + 1))
|
||||
sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
|
||||
info "checking mirror size ${sz} <= ${audit[$i]} bytes"
|
||||
if test "$sz" -le "${audit[$i]}"; then result "OK"; else
|
||||
result "mirror too big"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--min-mirror-bytes)
|
||||
i=$((i + 1))
|
||||
sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
|
||||
info "checking mirror size ${sz} >= ${audit[$i]} bytes"
|
||||
if test "$sz" -ge "${audit[$i]}"; then result "OK"; else
|
||||
result "mirror too small"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--file-matches)
|
||||
path="${audit[$((i + 1))]}"
|
||||
i=$((i + 2))
|
||||
info "checking ${path} matches ${audit[$i]}"
|
||||
if grep -aqE "${audit[$i]}" "${hostroot}/${path}"; then result "OK"; else
|
||||
result "no match"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--file-not-matches)
|
||||
path="${audit[$((i + 1))]}"
|
||||
i=$((i + 2))
|
||||
info "checking ${path} lacks ${audit[$i]}"
|
||||
if grep -aqE "${audit[$i]}" "${hostroot}/${path}"; then
|
||||
result "matched"
|
||||
exit 1
|
||||
else result "OK"; fi
|
||||
;;
|
||||
esac
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
@@ -14,8 +14,6 @@ stdlib only (http.server + ssl) -- no new build or runtime dependency.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
@@ -43,416 +41,6 @@ PAGE = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
||||
"""
|
||||
|
||||
|
||||
# --- /big/ seeded pseudo-site (36_local-bigcrawl) ---------------------------
|
||||
# Deterministic ~360-file tree; bodies derive from sha256(BIG_SEED, name) so
|
||||
# every run serves identical content and the test pins exact counts.
|
||||
BIG_SEED = "bigcrawl-lite-1"
|
||||
BIG_PAGES = 96
|
||||
BIG_FANOUT = 4
|
||||
# Fixed validator: a matching If-Modified-Since gets 304, so the update pass
|
||||
# revalidates instead of re-downloading.
|
||||
BIG_LASTMOD = "Mon, 01 Jan 2024 00:00:00 GMT"
|
||||
|
||||
BIG_CTYPES = {
|
||||
"html": "text/html",
|
||||
"css": "text/css",
|
||||
"js": "application/x-javascript",
|
||||
"png": "image/png",
|
||||
"gif": "image/gif",
|
||||
"jpg": "image/jpeg",
|
||||
"webp": "image/webp",
|
||||
"pdf": "application/pdf",
|
||||
"woff2": "font/woff2",
|
||||
"mp4": "video/mp4",
|
||||
"webm": "video/webm",
|
||||
"mp3": "audio/mpeg",
|
||||
"vtt": "text/vtt",
|
||||
"xml": "text/xml",
|
||||
"svg": "image/svg+xml",
|
||||
"jar": "application/java-archive",
|
||||
"bin": "application/octet-stream",
|
||||
}
|
||||
|
||||
# Honest magic bytes per claimed type so the #478 sniff never contests.
|
||||
BIG_MAGIC = {
|
||||
"png": b"\x89PNG\r\n\x1a\n",
|
||||
"gif": b"GIF89a",
|
||||
"jpg": b"\xff\xd8\xff\xe0",
|
||||
"webp": b"RIFF\x10\x27\x00\x00WEBPVP8 ",
|
||||
"pdf": b"%PDF-1.4\n",
|
||||
"woff2": b"wOF2",
|
||||
"mp4": b"\x00\x00\x00\x18ftypmp42",
|
||||
"webm": b"\x1a\x45\xdf\xa3",
|
||||
"mp3": b"ID3\x04\x00\x00\x00\x00\x00\x00",
|
||||
"jar": b"PK\x03\x04",
|
||||
}
|
||||
|
||||
|
||||
def big_blob(name, size):
|
||||
out = b""
|
||||
n = 0
|
||||
while len(out) < size:
|
||||
out += hashlib.sha256(f"{BIG_SEED}/{name}/{n}".encode()).digest()
|
||||
n += 1
|
||||
return out[:size]
|
||||
|
||||
|
||||
def big_asset(name):
|
||||
ext = name.rsplit(".", 1)[-1]
|
||||
size = 200 + int(hashlib.sha256(name.encode()).hexdigest(), 16) % 3800
|
||||
raw = big_blob(name, size)
|
||||
if ext in ("css", "js", "txt"):
|
||||
return b"/* " + raw.hex().encode() + b" */"
|
||||
return BIG_MAGIC.get(ext, b"") + raw
|
||||
|
||||
|
||||
def big_html(title, inner):
|
||||
page = (
|
||||
"<!DOCTYPE html><html><head><title>%s</title></head><body>\n%s\n</body></html>"
|
||||
% (
|
||||
title,
|
||||
inner,
|
||||
)
|
||||
)
|
||||
return page.encode()
|
||||
|
||||
|
||||
def _hexfill(name):
|
||||
return big_blob(name, 160).hex()
|
||||
|
||||
|
||||
HOME = '<a href="/big/index.html">home</a>'
|
||||
|
||||
BIG_TEXT_ASSETS = {
|
||||
"site.css": (
|
||||
"body { background: url(bg.png); } /* %s */" % _hexfill("site.css"),
|
||||
"text/css",
|
||||
),
|
||||
"print.css": ("p { margin: 0; } /* %s */" % _hexfill("print.css"), "text/css"),
|
||||
"blk.css": (
|
||||
'@import "blk2.css";\n'
|
||||
'@font-face { font-family: big; src: local("Nope Sans"), '
|
||||
'url(font.woff2) format("woff2"); }\n'
|
||||
"/* %s */" % _hexfill("blk.css"),
|
||||
"text/css",
|
||||
),
|
||||
# Absolute url() must come back relative after the rewrite (test greps it);
|
||||
# the \/ escapes collapse to an already-linked URL if taken literally.
|
||||
"blk2.css": (
|
||||
"body { background: url(/big/a/blk2-bg.png); }\n"
|
||||
"i { background: url(/big\\/a\\/bg.png); }\n"
|
||||
"/* %s */" % _hexfill("blk2.css"),
|
||||
"text/css",
|
||||
),
|
||||
# .open() grabs its first arg only (a method there is rejected, #218), so
|
||||
# the window.open single-URL form is the token-detected shape.
|
||||
"app.js": (
|
||||
'var im = new Image(); im.src = "/big/a/js-img.png";\n'
|
||||
'function pop() { window.open("/big/a/js-data.bin"); }\n'
|
||||
"// %s\n" % _hexfill("app.js"),
|
||||
"application/x-javascript",
|
||||
),
|
||||
"heavy.js": (
|
||||
'var h = new Image(); h.src = "/big/a/js1.png";\n'
|
||||
'function nav() { location.href = "/big/p/1.html"; }\n'
|
||||
'function pop() { window.open("/big/a/js2.bin"); }\n'
|
||||
"// %s\n" % _hexfill("heavy.js"),
|
||||
"application/x-javascript",
|
||||
),
|
||||
# text/javascript is fetched but never scanned: the URL inside must stay
|
||||
# out of the mirror.
|
||||
"decoy.js": (
|
||||
'var d = new Image(); d.src = "/big/x/never-scanned.png";\n',
|
||||
"text/javascript",
|
||||
),
|
||||
"subs.vtt": ("WEBVTT\n\n00:00.000 --> 00:01.000\nbig\n", "text/vtt"),
|
||||
"logo.svg": (
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="4" height="4">'
|
||||
'<image href="ref.png" width="4" height="4"/></svg>',
|
||||
"image/svg+xml",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _fam_feeds(port):
|
||||
return (
|
||||
'<link rel="alternate" type="application/rss+xml" href="/big/f12/rss.xml">'
|
||||
'<a href="/big/f12/atom.xml">atom</a>'
|
||||
'<a href="/big/f12/sitemap.xml">sitemap</a>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_plain(port):
|
||||
return (
|
||||
'<a href="../f1/one.html">one</a>'
|
||||
'<a href="./two.html">two</a>'
|
||||
'<a href="../../big/f1/tri.html">tri</a>'
|
||||
'<a href="/big/f1/abs.html">abs</a>'
|
||||
'<a href="/big/f1/list.html">list</a>'
|
||||
'<a href="/big/f1/list.html?page=2">p2</a>'
|
||||
'<a href="/big/f1/list.html?page=3&sort=asc">p3</a>'
|
||||
'<a href="/big/f1/dir">dir</a>'
|
||||
'<a href="">self</a><a href="#">frag</a>'
|
||||
'<a href="mailto:big@example.com">mail</a>'
|
||||
'<a href="tel:+15551234">tel</a>'
|
||||
'<a href="data:text/plain;base64,aGk=">data</a>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_srcset(port):
|
||||
return (
|
||||
'<img src="/big/a/f2-base.png">'
|
||||
'<img srcset="/big/a/f2-1x.png 1x, /big/a/f2-2x.png 2x"'
|
||||
' src="/big/a/f2-base.png">'
|
||||
'<img data-srcset="/big/a/f2-1x.png 1x, /big/a/f2-2x.png 2x"'
|
||||
' src="/big/a/f2-base.png" loading="lazy">'
|
||||
'<picture><source type="image/webp" srcset="/big/a/f2-alt.webp">'
|
||||
'<img src="/big/a/f2-base.png"></picture>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_media(port):
|
||||
return (
|
||||
'<video src="/big/a/clip.mp4" poster="/big/a/poster.jpg">'
|
||||
'<source src="/big/a/clip.webm" type="video/webm">'
|
||||
'<track src="/big/a/subs.vtt" kind="subtitles" srclang="en">'
|
||||
"</video>"
|
||||
'<audio><source src="/big/a/tune.mp3" type="audio/mpeg"></audio>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_css(port):
|
||||
# image-set with descriptors is a proven-safe decoy (engine-surface §6).
|
||||
return (
|
||||
'<link rel="stylesheet" href="/big/a/print.css" media="print">'
|
||||
'<div style="background:url(/big/a/attr-bg.png)">styled</div>'
|
||||
'<style>@import "/big/a/blk.css"; h1 { background: url(/big/a/blk-bg.gif); }'
|
||||
' h2 { background-image: image-set("/big/x/is1.png" 1x, "/big/x/is2.png" 2x); }'
|
||||
"</style>"
|
||||
)
|
||||
|
||||
|
||||
def _fam_js(port):
|
||||
# The concatenated string is rejected by the scanner (no single literal).
|
||||
return (
|
||||
'<script src="/big/a/heavy.js"></script>'
|
||||
'<script src="/big/a/decoy.js"></script>'
|
||||
"<script>document.write('<a href=\"/big/f5/dw.html\">dw</a>');\n"
|
||||
'var nope = "xx-" + "/big/x/concat.html";</script>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_meta(port):
|
||||
# Extensionless decoy targets stay unfetchable even if the aggressive
|
||||
# parser fires (no known extension, no scheme: rejected in every state).
|
||||
return (
|
||||
'<meta http-equiv="refresh" content="2;URL=/big/f6/refreshed.html">'
|
||||
'<a href="/big/f6/based.html">based</a>'
|
||||
'<meta property="og:image" content="/big/x/og">'
|
||||
'<meta name="twitter:image" content="/big/x/tw">'
|
||||
'<script type="application/ld+json">'
|
||||
'{"@type": "Thing", "image": "/big/x/jsonld.png"}</script>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_legacy(port):
|
||||
# Comma-valued applet archive is rejected whole by the engine (decoy).
|
||||
return (
|
||||
'<a href="/big/f7/frames.html">frames</a>'
|
||||
'<img src="/big/a/map.gif" usemap="#m">'
|
||||
'<map name="m">'
|
||||
'<area shape="rect" coords="0,0,9,9" href="/big/f7/area.html"></map>'
|
||||
'<embed src="/big/a/e.pdf" type="application/pdf" width="9" height="9">'
|
||||
'<object data="/big/a/o.pdf" type="application/pdf"></object>'
|
||||
'<applet archive="/big/x/aj.jar,/big/x/bj.jar" width="1" height="1"></applet>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_svg(port):
|
||||
return (
|
||||
'<svg width="9" height="9">'
|
||||
'<image href="/big/a/svg-in.png" width="4" height="4"/>'
|
||||
'<use xlink:href="#icon"/></svg>'
|
||||
'<img src="/big/a/logo.svg">'
|
||||
)
|
||||
|
||||
|
||||
def _fam_i18n(port):
|
||||
return (
|
||||
'<a href="/big/f9/caf%C3%A9.html">cafe</a>'
|
||||
'<a href="/big/f9/latin1.html">latin1</a>'
|
||||
'<a href="/big/f9/metaonly.html">meta</a>'
|
||||
'<a href="/big/f9/bom.html">bom</a>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_http(port):
|
||||
return (
|
||||
'<a href="/big/r/hop1">chain</a>'
|
||||
'<a href="/big/r/get42">get42</a>'
|
||||
'<a href="/big/d/01">d01</a>'
|
||||
'<a href="/big/d/02">d02</a>'
|
||||
'<a href="/big/f10/empty.html">empty</a>'
|
||||
'<a href="/big/d/dl">dl</a>'
|
||||
)
|
||||
|
||||
|
||||
def _fam_forms(port):
|
||||
# GET form action is rewritten but never fetched; formaction/ping are
|
||||
# outside the attribute tables (decoys).
|
||||
return (
|
||||
'<form action="/big/x/form-target.html" method="get">'
|
||||
'<input type="text" name="q">'
|
||||
'<input type="image" src="/big/a/btn.png" alt="go"></form>'
|
||||
'<a href="/big/f11/page.html">bare</a>'
|
||||
'<a href="/big/f11/page.html?utm_source=news&utm_medium=mail">utm</a>'
|
||||
'<a href="/big/f11/sess.html?PHPSESSID=deadbeef123">sess</a>'
|
||||
'<button formaction="/big/x/formact">go</button>'
|
||||
'<a href="/big/f11/page.html" ping="/big/x/ping">ping</a>'
|
||||
)
|
||||
|
||||
|
||||
BIG_FAMILIES = [
|
||||
_fam_feeds,
|
||||
_fam_plain,
|
||||
_fam_srcset,
|
||||
_fam_media,
|
||||
_fam_css,
|
||||
_fam_js,
|
||||
_fam_meta,
|
||||
_fam_legacy,
|
||||
_fam_svg,
|
||||
_fam_i18n,
|
||||
_fam_http,
|
||||
_fam_forms,
|
||||
]
|
||||
|
||||
|
||||
def big_link(m, style):
|
||||
return ["%d.html" % m, "../p/%d.html" % m, "/big/p/%d.html" % m][style]
|
||||
|
||||
|
||||
def big_page(n, port):
|
||||
style = n % 3
|
||||
home = ["../index.html", "/big/index.html", "../index.html"][style]
|
||||
parts = ['<a href="%s">home</a>' % home]
|
||||
if n > 0:
|
||||
parts.append('<a href="%s">up</a>' % big_link((n - 1) // BIG_FANOUT, style))
|
||||
for c in range(n * BIG_FANOUT + 1, n * BIG_FANOUT + BIG_FANOUT + 1):
|
||||
if c < BIG_PAGES:
|
||||
parts.append('<a href="%s">p%d</a>' % (big_link(c, style), c))
|
||||
parts.append('<link rel="stylesheet" href="/big/a/site.css">')
|
||||
parts.append('<script src="/big/a/app.js"></script>')
|
||||
exts = ["png", "gif", "jpg"]
|
||||
ia = "/big/a/i%da.%s" % (n, exts[n % 3])
|
||||
ib = "/big/a/i%db.%s" % (n, exts[(n + 1) % 3])
|
||||
# Rotate the second-image construct across deterministic table attributes.
|
||||
con = n % 4
|
||||
if con == 0:
|
||||
parts.append('<img src="%s"><img src="%s">' % (ia, ib))
|
||||
elif con == 1:
|
||||
parts.append(
|
||||
'<img src="%s"><table background="%s"><tr><td>t</td></tr></table>'
|
||||
% (ia, ib)
|
||||
)
|
||||
elif con == 2:
|
||||
parts.append('<img src="%s"><img src="%s" data-src="%s">' % (ia, ia, ib))
|
||||
else:
|
||||
parts.append(
|
||||
'<img src="%s" loading="lazy"><video poster="%s"></video>' % (ia, ib)
|
||||
)
|
||||
parts.append(BIG_FAMILIES[n % 12](port))
|
||||
return big_html("p%d" % n, "\n".join(parts))
|
||||
|
||||
|
||||
def big_index(port):
|
||||
return big_html(
|
||||
"big index",
|
||||
'<link rel="stylesheet" href="/big/a/site.css">'
|
||||
'<script src="/big/a/app.js"></script>'
|
||||
'<a href="p/0.html">root</a>'
|
||||
'<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
|
||||
'<a href="/big/f1/long.html?x=%s">long</a>'
|
||||
'<a href="/big/f1/gzok.html">gzok</a>'
|
||||
'<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
|
||||
'<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
|
||||
'<a href="/big/e/404.html">e404</a>'
|
||||
'<a href="/big/e/410.html">e410</a>'
|
||||
'<a href="/big/e/500.html">e500</a>'
|
||||
'<a href="/big/e/gztrunc.html">gzt</a>'
|
||||
'<a href="?">query</a>' % ("a" * 900, port, port),
|
||||
)
|
||||
|
||||
|
||||
BIG_REDIRECTS = {
|
||||
"/big/r/hop1": (301, "/big/r/hop2"),
|
||||
"/big/r/hop2": (302, "/big/f10/land.html"),
|
||||
"/big/r/get42": (301, "/big/a/doc.pdf"),
|
||||
"/big/f1/dir": (301, "/big/f1/dir/"),
|
||||
}
|
||||
|
||||
BIG_SIMPLE_PAGES = {
|
||||
"/big/p/two.html": "dot-slash target",
|
||||
"/big/f1/one.html": "one",
|
||||
"/big/f1/tri.html": "tri",
|
||||
"/big/f1/abs.html": "abs",
|
||||
"/big/f1/dir/": "dir index",
|
||||
"/big/f1/long.html": "long",
|
||||
"/big/f1/gzok.html": "gzok",
|
||||
"/big/f1/protorel.html": "protorel",
|
||||
"/big/f1/abshost.html": "abshost",
|
||||
"/big/f5/dw.html": "dw target",
|
||||
"/big/f6/refreshed.html": "refreshed",
|
||||
"/big/f6/sub/leaf.html": "leaf",
|
||||
"/big/f7/fa.html": "frame a",
|
||||
"/big/f7/fb.html": "frame b",
|
||||
"/big/f7/fn.html": "noframes",
|
||||
"/big/f7/area.html": "area",
|
||||
"/big/f10/land.html": "landed",
|
||||
"/big/f11/page.html": "the page",
|
||||
"/big/f11/sess.html": "the sess page",
|
||||
}
|
||||
|
||||
# Extensionless downloads: name resolution is wire-type driven (#478 contract).
|
||||
BIG_DOWNLOADS = {
|
||||
"/big/d/01": ("pdf", None),
|
||||
"/big/d/02": ("png", None),
|
||||
"/big/d/dl": ("pdf", 'attachment; filename="named.pdf"'),
|
||||
}
|
||||
|
||||
|
||||
def _big_rss(port):
|
||||
# purl.org marker makes the feed parse; item URLs are already-linked pages.
|
||||
return (
|
||||
'<?xml version="1.0"?>\n'
|
||||
'<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">\n'
|
||||
"<channel><title>big</title><link>http://127.0.0.1:%d/big/index.html</link>\n"
|
||||
"<item><title>i1</title><link>http://127.0.0.1:%d/big/p/1.html</link>\n"
|
||||
'<enclosure url="http://127.0.0.1:%d/big/p/2.html" type="text/html"/></item>\n'
|
||||
"</channel></rss>\n" % (port, port, port)
|
||||
).encode()
|
||||
|
||||
|
||||
def _big_atom(port):
|
||||
# No purl marker: emitted verbatim, its URL must never be fetched.
|
||||
return (
|
||||
'<?xml version="1.0"?>\n'
|
||||
'<feed xmlns="http://www.w3.org/2005/Atom"><title>big</title>\n'
|
||||
"<entry><title>e1</title>"
|
||||
'<link href="http://127.0.0.1:%d/big/x/atom-only.html"/>'
|
||||
"</entry></feed>\n" % port
|
||||
).encode()
|
||||
|
||||
|
||||
def _big_sitemap(port):
|
||||
return (
|
||||
'<?xml version="1.0"?>\n'
|
||||
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
|
||||
"<url><loc>http://127.0.0.1:%d/big/x/sitemap-only.html</loc></url>\n"
|
||||
"</urlset>\n" % port
|
||||
).encode()
|
||||
|
||||
|
||||
class Handler(SimpleHTTPRequestHandler):
|
||||
# Quieter logging; the launcher captures httrack's own log anyway.
|
||||
def log_message(self, fmt, *args):
|
||||
@@ -546,14 +134,12 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
|
||||
# --- type/extension matrix (issue #267 family) -------------------------
|
||||
|
||||
def send_raw(self, body, content_type, extra_headers=()):
|
||||
def send_raw(self, body, content_type):
|
||||
"""Send a raw body with an explicit Content-Type, or none at all when
|
||||
content_type is None (to observe httrack's typeless-file naming)."""
|
||||
self.send_response(200)
|
||||
if content_type is not None:
|
||||
self.send_header("Content-Type", content_type)
|
||||
for name, value in extra_headers:
|
||||
self.send_header(name, value)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
@@ -562,8 +148,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
# Fake-binary blobs for the image/pdf/typeless cases.
|
||||
FAKE_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 64
|
||||
FAKE_PDF = b"%PDF-1.4\n" + b"\x00" * 64
|
||||
FAKE_JPEG = b"\xff\xd8\xff\xe0" + b"\x00" * 64
|
||||
BIG_JPEG = b"\xff\xd8\xff\xe0" + bytes(range(256)) * 64 # > sniff window
|
||||
|
||||
# path -> (body, content_type); None sends no header, "" sends an empty
|
||||
# Content-Type value (no usable type, must be treated like None).
|
||||
@@ -575,8 +159,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/types/notype.pdf": (FAKE_PDF, None),
|
||||
"/types/emptyct.png": (FAKE_PNG, ""),
|
||||
"/types/lie.png": (FAKE_PNG, "text/html"),
|
||||
"/types/wrongtype.jpg": (FAKE_JPEG, "image/png"),
|
||||
"/types/bigtype.jpg": (BIG_JPEG, "image/png"),
|
||||
"/types/report.pdf": (b"<html><body>real page</body></html>", "text/html"),
|
||||
"/types/page.htm": (b"<html><body>htm page</body></html>", "text/html"),
|
||||
"/types/script.js": (b"var x = 1;\n", "application/javascript"),
|
||||
@@ -594,10 +176,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
'\t<a href="notype.pdf">notypepdf</a>\n'
|
||||
'\t<img src="emptyct.png" />\n'
|
||||
'\t<img src="lie.png" />\n'
|
||||
'\t<img src="wrongtype.jpg" />\n'
|
||||
'\t<img src="bigtype.jpg" />\n'
|
||||
'\t<img src="mutant.jpg" />\n'
|
||||
'\t<img src="packed.jpg" />\n'
|
||||
'\t<a href="report.pdf">report</a>\n'
|
||||
'\t<a href="page.htm">htm</a>\n'
|
||||
'\t<script src="script.js"></script>\n'
|
||||
@@ -612,25 +190,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
body, ctype = self.TYPE_MATRIX[path]
|
||||
self.send_raw(body, ctype)
|
||||
|
||||
# content changes between crawls: run 1 sniffs JPEG, the update pass must
|
||||
# keep the run-1 name (recorded verdict) even though the body is now PNG
|
||||
MUTANT_SEEN = set()
|
||||
|
||||
def route_types_mutant(self):
|
||||
path = urlsplit(self.path).path
|
||||
body = self.FAKE_PNG if path in self.MUTANT_SEEN else self.FAKE_JPEG
|
||||
if self.command != "HEAD":
|
||||
self.MUTANT_SEEN.add(path)
|
||||
self.send_raw(body, "image/png")
|
||||
|
||||
# gzip on the wire: the sniff must see the decoded body, not the stream
|
||||
def route_types_packed(self):
|
||||
self.send_raw(
|
||||
gzip.compress(self.FAKE_JPEG),
|
||||
"image/png",
|
||||
extra_headers=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
|
||||
# --- MIME-type exclusion abort (issue #58) -----------------------------
|
||||
# A -mime:application/pdf filter must abort the transfer once the header
|
||||
# arrives, not download the whole body and discard it.
|
||||
@@ -781,7 +340,7 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
self.send_raw(b"", "text/html")
|
||||
|
||||
# broken Content-Length (#32/#41): declared size != bytes sent. httrack
|
||||
# warns "incomplete transfer" and skips the cache unless -%B.
|
||||
# warns "bogus state (broken size)" and skips the cache unless -%B.
|
||||
def route_size_index(self):
|
||||
self.send_html('\t<a href="oversize.bin">over</a>\n')
|
||||
|
||||
@@ -795,27 +354,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
# Content-Disposition naming: the attachment filename replaces the
|
||||
# URL-derived name; path components in it are stripped (RFC 2616).
|
||||
CDISPO_NAMES = {
|
||||
"/cdispo/fetch.php": "report.pdf",
|
||||
"/cdispo/evil.php": "../../evil.pdf",
|
||||
}
|
||||
|
||||
def route_cdispo_index(self):
|
||||
self.send_html(
|
||||
'\t<a href="fetch.php">report</a>\n' '\t<a href="evil.php">evil</a>\n'
|
||||
)
|
||||
|
||||
def route_cdispo(self):
|
||||
filename = self.CDISPO_NAMES[urlsplit(self.path).path]
|
||||
cdispo = 'attachment; filename="%s"' % filename
|
||||
self.send_raw(
|
||||
self.FAKE_PDF,
|
||||
"application/pdf",
|
||||
extra_headers=[("Content-Disposition", cdispo)],
|
||||
)
|
||||
|
||||
# 302 whose Location carries a #fragment (#204): the fragment is a UA anchor
|
||||
# that must be dropped before the target is fetched. A leaked '#' reaches the
|
||||
# strict-server guard below and 400s.
|
||||
@@ -831,97 +369,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
def route_redir_target(self):
|
||||
self.send_raw(b"<html><body>redirect target</body></html>\n", "text/html")
|
||||
|
||||
# --- /mini304/: tiny fully-cacheable site (an update gets only 304s) ---
|
||||
def route_mini304_index(self):
|
||||
self.big_send(
|
||||
b'<html><body>\n\t<a href="page.html">page</a>\n</body></html>\n',
|
||||
"text/html",
|
||||
)
|
||||
|
||||
def route_mini304_page(self):
|
||||
self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
|
||||
|
||||
# --- delayed-type degenerate paths (issues #5/#107) --------------------
|
||||
def route_delayed_index(self):
|
||||
self.send_html(
|
||||
'\t<a href="noloc.php">noloc</a>\n'
|
||||
'\t<a href="selfloop.php">selfloop</a>\n'
|
||||
'\t<a href="chain1.php">chain</a>\n'
|
||||
'\t<a href="redir.php">redir</a>\n'
|
||||
'\t<a href="notype.bin">notype</a>\n'
|
||||
'\t<a href="empty.php">empty</a>\n'
|
||||
)
|
||||
|
||||
def send_redirect(self, location):
|
||||
self.send_response(302, "Found")
|
||||
if location is not None:
|
||||
self.send_header("Location", location)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def route_delayed_noloc(self):
|
||||
self.send_redirect(None) # 302 without Location: name never resolves
|
||||
|
||||
def route_delayed_selfloop(self):
|
||||
self.send_redirect("selfloop.php")
|
||||
|
||||
def route_delayed_chain(self):
|
||||
# chain1..chain9: one more hop than the type-check redirect budget
|
||||
n = int(urlsplit(self.path).path.rsplit("chain", 1)[1].split(".")[0])
|
||||
if n < 9:
|
||||
self.send_redirect("chain%d.php" % (n + 1))
|
||||
else:
|
||||
self.send_raw(self.FAKE_PDF, "application/pdf")
|
||||
|
||||
def route_delayed_redir(self):
|
||||
self.send_redirect("real.pdf")
|
||||
|
||||
def route_delayed_realpdf(self):
|
||||
self.send_raw(self.FAKE_PDF, "application/pdf")
|
||||
|
||||
def route_delayed_notype(self):
|
||||
self.send_raw(self.FAKE_PDF, None)
|
||||
|
||||
def route_delayed_empty(self):
|
||||
self.send_raw(b"", "text/html") # 200 + Content-Length: 0
|
||||
|
||||
# -E time-limit (#481): pages that trickle far longer than any -E budget,
|
||||
# so only an engine-side abort can end the crawl.
|
||||
TRICKLE_SECONDS = 60
|
||||
|
||||
def send_bin_index(self):
|
||||
"""Index page linking p0.bin..p7.bin (shared by trickle and bigfiles)."""
|
||||
self.send_html(
|
||||
"".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
|
||||
)
|
||||
|
||||
def route_trickle_index(self):
|
||||
self.send_bin_index()
|
||||
|
||||
def route_trickle_page(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(2 * self.TRICKLE_SECONDS))
|
||||
self.end_headers()
|
||||
if self.command == "HEAD":
|
||||
return
|
||||
try:
|
||||
for _ in range(self.TRICKLE_SECONDS):
|
||||
self.wfile.write(b"xy")
|
||||
self.wfile.flush()
|
||||
time.sleep(1.0)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# -M byte cap (#77): large fast files so a crawl overruns -M immediately.
|
||||
BIGFILE_BYTES = 640 * 1024
|
||||
|
||||
def route_bigfiles_index(self):
|
||||
self.send_bin_index()
|
||||
|
||||
def route_bigfile(self):
|
||||
self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
|
||||
|
||||
ROUTES = {
|
||||
"/cookies/entrance.php": route_entrance,
|
||||
"/cookies/second.php": route_second,
|
||||
@@ -937,10 +384,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/types/notype.pdf": route_types,
|
||||
"/types/emptyct.png": route_types,
|
||||
"/types/lie.png": route_types,
|
||||
"/types/wrongtype.jpg": route_types,
|
||||
"/types/bigtype.jpg": route_types,
|
||||
"/types/mutant.jpg": route_types_mutant,
|
||||
"/types/packed.jpg": route_types_packed,
|
||||
"/types/report.pdf": route_types,
|
||||
"/types/page.htm": route_types,
|
||||
"/types/script.js": route_types,
|
||||
@@ -963,190 +406,11 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/mimex/index.html": route_mimex_index,
|
||||
"/mimex/blob.pdf": route_mimex_blob,
|
||||
"/mimex/real.html": route_mimex_real,
|
||||
"/cdispo/index.html": route_cdispo_index,
|
||||
"/cdispo/fetch.php": route_cdispo,
|
||||
"/cdispo/evil.php": route_cdispo,
|
||||
"/delayed/index.html": route_delayed_index,
|
||||
"/trickle/index.html": route_trickle_index,
|
||||
"/trickle/p0.bin": route_trickle_page,
|
||||
"/trickle/p1.bin": route_trickle_page,
|
||||
"/trickle/p2.bin": route_trickle_page,
|
||||
"/trickle/p3.bin": route_trickle_page,
|
||||
"/trickle/p4.bin": route_trickle_page,
|
||||
"/trickle/p5.bin": route_trickle_page,
|
||||
"/trickle/p6.bin": route_trickle_page,
|
||||
"/trickle/p7.bin": route_trickle_page,
|
||||
"/bigfiles/index.html": route_bigfiles_index,
|
||||
"/bigfiles/p0.bin": route_bigfile,
|
||||
"/bigfiles/p1.bin": route_bigfile,
|
||||
"/bigfiles/p2.bin": route_bigfile,
|
||||
"/bigfiles/p3.bin": route_bigfile,
|
||||
"/bigfiles/p4.bin": route_bigfile,
|
||||
"/bigfiles/p5.bin": route_bigfile,
|
||||
"/bigfiles/p6.bin": route_bigfile,
|
||||
"/bigfiles/p7.bin": route_bigfile,
|
||||
"/delayed/noloc.php": route_delayed_noloc,
|
||||
"/delayed/selfloop.php": route_delayed_selfloop,
|
||||
"/delayed/redir.php": route_delayed_redir,
|
||||
"/delayed/real.pdf": route_delayed_realpdf,
|
||||
"/delayed/notype.bin": route_delayed_notype,
|
||||
"/delayed/empty.php": route_delayed_empty,
|
||||
"/delayed/chain1.php": route_delayed_chain,
|
||||
"/delayed/chain2.php": route_delayed_chain,
|
||||
"/delayed/chain3.php": route_delayed_chain,
|
||||
"/delayed/chain4.php": route_delayed_chain,
|
||||
"/delayed/chain5.php": route_delayed_chain,
|
||||
"/delayed/chain6.php": route_delayed_chain,
|
||||
"/delayed/chain7.php": route_delayed_chain,
|
||||
"/delayed/chain8.php": route_delayed_chain,
|
||||
"/delayed/chain9.php": route_delayed_chain,
|
||||
"/redir/index.html": route_redir_index,
|
||||
"/redir/go.php": route_redir_go,
|
||||
"/redir/target.html": route_redir_target,
|
||||
"/mini304/index.html": route_mini304_index,
|
||||
"/mini304/page.html": route_mini304_page,
|
||||
}
|
||||
|
||||
# --- /big/ seeded pseudo-site ------------------------------------------
|
||||
|
||||
def big_send(self, body, ctype, code=200, extra=()):
|
||||
if code == 200 and self.headers.get("If-Modified-Since") == BIG_LASTMOD:
|
||||
self.send_response(304)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
self.send_response(code)
|
||||
if code == 200:
|
||||
self.send_header("Last-Modified", BIG_LASTMOD)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
for name, value in extra:
|
||||
self.send_header(name, value)
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def big_error(self, code, reason):
|
||||
body = big_html("error", "<p>%d</p>%s" % (code, HOME))
|
||||
self.big_send(body, "text/html", code=code, extra=[("X-Reason", reason)])
|
||||
|
||||
def route_big(self):
|
||||
split = urlsplit(self.path)
|
||||
path = unquote(split.path)
|
||||
port = self.server.server_address[1]
|
||||
if path in BIG_REDIRECTS:
|
||||
code, location = BIG_REDIRECTS[path]
|
||||
self.send_response(code)
|
||||
self.send_header("Location", location)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
elif path == "/big/index.html":
|
||||
self.big_send(big_index(port), "text/html")
|
||||
elif path in BIG_SIMPLE_PAGES:
|
||||
body = big_html(path, "<p>%s</p>%s" % (BIG_SIMPLE_PAGES[path], HOME))
|
||||
if path == "/big/f1/gzok.html":
|
||||
self.big_send(
|
||||
gzip.compress(body, mtime=0),
|
||||
"text/html",
|
||||
extra=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
else:
|
||||
self.big_send(body, "text/html")
|
||||
elif path == "/big/f1/list.html":
|
||||
# Pagination: distinct content per query string.
|
||||
body = big_html("list", "<p>listing %s</p>%s" % (split.query or "1", HOME))
|
||||
self.big_send(body, "text/html")
|
||||
elif path == "/big/f6/based.html":
|
||||
self.big_send(
|
||||
big_html(
|
||||
"based",
|
||||
'<base href="http://127.0.0.1:%d/big/f6/sub/">'
|
||||
'<a href="leaf.html">leaf</a>' % port,
|
||||
),
|
||||
"text/html",
|
||||
)
|
||||
elif path == "/big/f7/frames.html":
|
||||
self.big_send(
|
||||
b'<html><frameset cols="50%,50%"><frame src="fa.html">'
|
||||
b'<frame src="fb.html"><noframes><body><a href="fn.html">fn</a>'
|
||||
b"</body></noframes></frameset></html>",
|
||||
"text/html",
|
||||
)
|
||||
elif path == "/big/f9/café.html":
|
||||
self.big_send(big_html("cafe", "<p>cafe</p>%s" % HOME), "text/html")
|
||||
elif path == "/big/f9/latin1.html":
|
||||
self.big_send(
|
||||
b"<html><body><p>caf\xe9 latin</p></body></html>",
|
||||
"text/html; charset=ISO-8859-1",
|
||||
)
|
||||
elif path == "/big/f9/metaonly.html":
|
||||
self.big_send(
|
||||
'<html><head><meta charset="utf-8"></head>'
|
||||
"<body><p>café meta</p></body></html>".encode(),
|
||||
"text/html",
|
||||
)
|
||||
elif path == "/big/f9/bom.html":
|
||||
self.big_send(
|
||||
b"\xef\xbb\xbf" + big_html("bom", "<p>bom</p>%s" % HOME), "text/html"
|
||||
)
|
||||
elif path == "/big/f10/empty.html":
|
||||
self.big_send(b"", "text/html")
|
||||
elif path == "/big/f12/rss.xml":
|
||||
self.big_send(_big_rss(port), "text/xml")
|
||||
elif path == "/big/f12/atom.xml":
|
||||
self.big_send(_big_atom(port), "application/xml")
|
||||
elif path == "/big/f12/sitemap.xml":
|
||||
self.big_send(_big_sitemap(port), "text/xml")
|
||||
elif path.startswith("/big/p/"):
|
||||
try:
|
||||
n = int(path[len("/big/p/") : -len(".html")])
|
||||
except ValueError:
|
||||
n = -1
|
||||
if 0 <= n < BIG_PAGES and path.endswith(".html"):
|
||||
self.big_send(big_page(n, port), "text/html")
|
||||
else:
|
||||
self.big_error(404, "no such page")
|
||||
elif path.startswith("/big/a/") or path.startswith("/big/x/"):
|
||||
name = path[len("/big/a/") :]
|
||||
if path.startswith("/big/a/") and name in BIG_TEXT_ASSETS:
|
||||
text, ctype = BIG_TEXT_ASSETS[name]
|
||||
self.big_send(text.encode(), ctype)
|
||||
elif name.endswith(".html"):
|
||||
# Decoy targets 200 so a parser leak becomes a mirror file.
|
||||
self.big_send(big_html(name, "<p>%s</p>" % name), "text/html")
|
||||
else:
|
||||
ext = name.rsplit(".", 1)[-1]
|
||||
ctype = BIG_CTYPES.get(ext, "application/octet-stream")
|
||||
self.big_send(big_asset(name), ctype)
|
||||
elif path in BIG_DOWNLOADS:
|
||||
ext, cdispo = BIG_DOWNLOADS[path]
|
||||
extra = [("Content-Disposition", cdispo)] if cdispo else []
|
||||
self.big_send(
|
||||
big_asset(path[len("/big/") :] + "." + ext),
|
||||
BIG_CTYPES[ext],
|
||||
extra=extra,
|
||||
)
|
||||
elif path == "/big/e/404.html":
|
||||
self.big_error(404, "Not Found")
|
||||
elif path == "/big/e/410.html":
|
||||
self.big_error(410, "Gone")
|
||||
elif path == "/big/e/500.html":
|
||||
self.big_error(500, "Server Error")
|
||||
elif path == "/big/e/gztrunc.html":
|
||||
# Half a gzip stream, honest Content-Length: decode fails, and the
|
||||
# missing Last-Modified keeps it the one uncacheable resource.
|
||||
full = gzip.compress(big_html("gz", "x" * 3000), mtime=0)
|
||||
body = full[: len(full) // 2]
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Encoding", "gzip")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
else:
|
||||
self.big_error(404, "no such big path")
|
||||
|
||||
# --- dispatch ----------------------------------------------------------
|
||||
|
||||
def reject_fragment(self):
|
||||
@@ -1162,9 +426,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
def dispatch(self):
|
||||
self._set_cookies = []
|
||||
path = urlsplit(self.path).path
|
||||
if path.startswith("/big/"):
|
||||
self.route_big()
|
||||
return True
|
||||
# Match percent-encoded paths (accented #157 route) by their decoded form.
|
||||
handler = self.ROUTES.get(path) or self.ROUTES.get(unquote(path))
|
||||
if handler is not None:
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
<html><body>
|
||||
<a href=target.html#sec>unquoted fragment link</a>
|
||||
<a href="target.html#sec2">quoted fragment link</a>
|
||||
</body></html>
|
||||
@@ -1 +0,0 @@
|
||||
<html><body><a name="sec"></a><a name="sec2"></a>target</body></html>
|
||||
@@ -211,9 +211,7 @@ main() {
|
||||
# lintian ourselves below as the real gate.
|
||||
local -a debuild_opts=(--no-lintian)
|
||||
local -a build_opts=()
|
||||
# -d: a source build runs no debhelper, so don't require Build-Depends
|
||||
# locally (the buildds and the --sbuild gate enforce them).
|
||||
[[ $source_only -eq 1 ]] && build_opts+=(-S -d)
|
||||
[[ $source_only -eq 1 ]] && build_opts+=(-S)
|
||||
if [[ $unsigned -eq 1 ]]; then
|
||||
build_opts+=(-us -uc)
|
||||
else
|
||||
@@ -236,15 +234,12 @@ main() {
|
||||
|
||||
# The real lintian gate (debuild only reports, it does not fail on tags).
|
||||
# --profile debian: CI runners are Ubuntu, whose vendor data would wrongly
|
||||
# reject the Debian "unstable" distribution. Suppressed tags are stale-local-
|
||||
# lintian skew, not package defects: newer-standards-version, and
|
||||
# recommended-field (old lintian still wants the Priority field the sid
|
||||
# lintian in CI accepts dropping). set -e turns any error/warning tag into
|
||||
# a failure.
|
||||
# reject the Debian "unstable" distribution. newer-standards-version only
|
||||
# means the local lintian is older than the buildds', not a package
|
||||
# defect, so suppress it. set -e turns any error/warning tag into a failure.
|
||||
info "running lintian gate (--fail-on=error,warning)"
|
||||
lintian --profile debian -I -i --fail-on=error,warning \
|
||||
--suppress-tags newer-standards-version,recommended-field \
|
||||
"${changes[@]}"
|
||||
--suppress-tags newer-standards-version "${changes[@]}"
|
||||
|
||||
dcmd cp -- "${changes[@]}" "$outdir/"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user