mirror of
https://github.com/xroche/httrack.git
synced 2026-07-27 19:12:54 +03:00
Compare commits
19 Commits
fix/refetc
...
fix/ftp-re
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5751991897 | ||
|
|
95f8ebaa44 | ||
|
|
3a096c589a | ||
|
|
1a4a5082b7 | ||
|
|
5378fe529a | ||
|
|
d7312da326 | ||
|
|
1ad59352f2 | ||
|
|
4fd6767a8f | ||
|
|
1ef3458b2e | ||
|
|
56389103bf | ||
|
|
a4db58f7b7 | ||
|
|
1f9728d816 | ||
|
|
99bd9bb674 | ||
|
|
29dfd2df59 | ||
|
|
55765815f5 | ||
|
|
869b8479e9 | ||
|
|
9fe47c3986 | ||
|
|
37fa549ac5 | ||
|
|
de8c0eebfc |
5
.github/workflows/windows-build.yml
vendored
5
.github/workflows/windows-build.yml
vendored
@@ -225,11 +225,12 @@ jobs:
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
|
||||
# footer-overflow and purge-longpath skip on Windows (need a path past MAX_PATH);
|
||||
# crange pending #581;
|
||||
# webdav-mime needs a reapable background listener, which MSYS cannot give it;
|
||||
# badmtime needs a filesystem that stores an mtime past gmtime's range;
|
||||
# single-file ends on a GUI half needing htsserver, which this job does not build.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
|
||||
expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
@@ -487,141 +487,6 @@ regen:
|
||||
|
||||
#define HTS_DATA_UNKNOWN_HTML_LEN 0
|
||||
|
||||
#define HTS_DATA_ERROR_HTML "<html>"LF\
|
||||
"<html xmlns=\"http://www.w3.org/1999/xhtml\" lang=\"en\">"LF\
|
||||
""LF\
|
||||
"<head>"LF\
|
||||
" <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />"LF\
|
||||
" <meta name=\"description\" content=\"HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help\" />"LF\
|
||||
" <meta name=\"keywords\" content=\"httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software\" />"LF\
|
||||
" <title>Page not retrieved! - HTTrack Website Copier</title>"LF\
|
||||
" <style type=\"text/css\">"LF\
|
||||
" <!--"LF\
|
||||
""LF\
|
||||
"body {"LF\
|
||||
" margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;"LF\
|
||||
" background: #77b;"LF\
|
||||
"}"LF\
|
||||
"body, td {"LF\
|
||||
" font: 14px \"Trebuchet MS\", Verdana, Arial, Helvetica, sans-serif;"LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
"#subTitle {"LF\
|
||||
" background: #000; color: #fff; padding: 4px; font-weight: bold; "LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
"#siteNavigation a, #siteNavigation .current {"LF\
|
||||
" font-weight: bold; color: #448;"LF\
|
||||
" }"LF\
|
||||
"#siteNavigation a:link { text-decoration: none; }"LF\
|
||||
"#siteNavigation a:visited { text-decoration: none; }"LF\
|
||||
""LF\
|
||||
"#siteNavigation .current { background-color: #ccd; }"LF\
|
||||
""LF\
|
||||
"#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }"LF\
|
||||
"#siteNavigation a:active { text-decoration: none; background-color: #ccc; }"LF\
|
||||
""LF\
|
||||
""LF\
|
||||
"a:link { text-decoration: underline; color: #00f; }"LF\
|
||||
"a:visited { text-decoration: underline; color: #000; }"LF\
|
||||
"a:hover { text-decoration: underline; color: #c00; }"LF\
|
||||
"a:active { text-decoration: underline; }"LF\
|
||||
""LF\
|
||||
"#pageContent {"LF\
|
||||
" clear: both;"LF\
|
||||
" border-bottom: 6px solid #000;"LF\
|
||||
" padding: 10px; padding-top: 20px;"LF\
|
||||
" line-height: 1.65em;"LF\
|
||||
" background-image: url(backblue.gif);"LF\
|
||||
" background-repeat: no-repeat;"LF\
|
||||
" background-position: top right;"LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
"#pageContent, #siteNavigation {"LF\
|
||||
" background-color: #ccd;"LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
""LF\
|
||||
".imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }"LF\
|
||||
".imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }"LF\
|
||||
""LF\
|
||||
"hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }"LF\
|
||||
""LF\
|
||||
"h1 { margin: 0; font-weight: bold; font-size: 2em; }"LF\
|
||||
"h2 { margin: 0; font-weight: bold; font-size: 1.6em; }"LF\
|
||||
"h3 { margin: 0; font-weight: bold; font-size: 1.3em; }"LF\
|
||||
"h4 { margin: 0; font-weight: bold; font-size: 1.18em; }"LF\
|
||||
""LF\
|
||||
".blak { background-color: #000; }"LF\
|
||||
".hide { display: none; }"LF\
|
||||
".tableWidth { min-width: 400px; }"LF\
|
||||
""LF\
|
||||
".tblRegular { border-collapse: collapse; }"LF\
|
||||
".tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }"LF\
|
||||
".tblHeaderColor, .tblHeaderColor td { background: #99c; }"LF\
|
||||
".tblNoBorder td { border: 0; }"LF\
|
||||
""LF\
|
||||
""LF\
|
||||
"// -->"LF\
|
||||
"</style>"LF\
|
||||
""LF\
|
||||
"</head>"LF\
|
||||
""LF\
|
||||
"<table width=\"76%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"3\" class=\"tableWidth\">"LF\
|
||||
" <tr>"LF\
|
||||
" <td id=\"subTitle\">HTTrack Website Copier - Open Source offline browser</td>"LF\
|
||||
" </tr>"LF\
|
||||
"</table>"LF\
|
||||
"<table width=\"76%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"0\" class=\"tableWidth\">"LF\
|
||||
"<tr class=\"blak\">"LF\
|
||||
"<td>"LF\
|
||||
" <table width=\"100%%\" border=\"0\" align=\"center\" cellspacing=\"1\" cellpadding=\"0\">"LF\
|
||||
" <tr>"LF\
|
||||
" <td colspan=\"6\"> "LF\
|
||||
" <table width=\"100%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"10\">"LF\
|
||||
" <tr> "LF\
|
||||
" <td id=\"pageContent\"> "LF\
|
||||
"<!-- ==================== End prologue ==================== -->"LF\
|
||||
"<h1><strong><u>Oops!...</u></strong></h1>"LF\
|
||||
"<h3>This page has <font color=\"red\"><em>not</em></font> been retrieved by HTTrack Website Copier (%s). </h3>"LF\
|
||||
"<script language=\"Javascript\">"LF\
|
||||
"<!--"LF\
|
||||
" var loc=document.location.toString();"LF\
|
||||
" if (loc) {"LF\
|
||||
" var pos=loc.indexOf('link=');"LF\
|
||||
" if (pos>0) {"LF\
|
||||
" document.write('Clic to the link <b>below</b> to go to the online location!<br><a href=\"'+loc.substring(pos+5)+'\">'+loc.substring(pos+5)+'</a><br>');"LF\
|
||||
" } else"LF\
|
||||
" document.write('(no location defined)');"LF\
|
||||
" }"LF\
|
||||
"// -->"LF\
|
||||
"</script>"LF\
|
||||
"<h6 align=\"right\">Mirror by HTTrack Website Copier</h6>"LF\
|
||||
"</body>"LF\
|
||||
"</html>"LF\
|
||||
"<!-- ==================== Start epilogue ==================== -->"LF\
|
||||
" </td>"LF\
|
||||
" </tr>"LF\
|
||||
" </table>"LF\
|
||||
" </td>"LF\
|
||||
" </tr>"LF\
|
||||
" </table>"LF\
|
||||
"</td>"LF\
|
||||
"</tr>"LF\
|
||||
"</table>"LF\
|
||||
""LF\
|
||||
"<table width=\"76%%\" height=\"100%%\" border=\"0\" align=\"center\" valign=\"bottom\" cellspacing=\"0\" cellpadding=\"0\">"LF\
|
||||
" <tr>"LF\
|
||||
" <td id=\"footer\"><small>© 2014 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\
|
||||
" </tr>"LF\
|
||||
"</table>"LF\
|
||||
""LF\
|
||||
"</body>"LF\
|
||||
""LF\
|
||||
"</html>"LF\
|
||||
""LF\
|
||||
""LF
|
||||
|
||||
// image gif "unknown"
|
||||
#define HTS_DATA_UNKNOWN_GIF \
|
||||
"\x47\x49\x46\x38\x39\x61\x20\x0\x20\x0\xf7\xff\x0\xc0\xc0\xc0\xff\x0\x0\xfc\x3\x0\xf8\x6\x0\xf6\x9\x0\xf2\xc\x0\xf0\xf\x0\xf0\xe\x0\xed\x11\x0\xec\x13\x0\xeb\x14\x0\xe9\x15\x0\xe8\x18\x0\xe6\x18\x0\xe5\x1a\x0\xe3\x1c\x0\xe2\x1d\x0\xe1\x1e\x0\xdf\x20\x0\xdd\x23\x0\xdd\x22\x0\xdb\x23\x0\xda\x25\x0\xd9\x25\x0\xd8\x27\x0\xd6\x29\x0\xd5\x2a\x0\xd3\x2c\x0\xd2\x2d\x0"\
|
||||
|
||||
116
src/htsback.c
116
src/htsback.c
@@ -340,12 +340,61 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr
|
||||
}
|
||||
|
||||
static int slot_can_be_cached_on_disk(const lien_back * back) {
|
||||
/* A pending backup or spool means the slot is not finalized, and the swap
|
||||
would unlink it through back_clear_entry() (#771). */
|
||||
if (back->tmpfile != NULL && back->tmpfile[0] != '\0')
|
||||
return 0;
|
||||
return (back->status == STATUS_READY && back->locked == 0
|
||||
&& back->url_sav[0] != '\0'
|
||||
&& strcmp(back->url_sav, BACK_ADD_TEST) != 0);
|
||||
/* Note: not checking !IS_DELAYED_EXT(back->url_sav) or it will quickly cause the slots to be filled! */
|
||||
}
|
||||
|
||||
int back_selftest_slot_swap(void) {
|
||||
lien_back back;
|
||||
int err = 0;
|
||||
|
||||
#define CHECK(want, why) \
|
||||
do { \
|
||||
if (slot_can_be_cached_on_disk(&back) != (want)) { \
|
||||
fprintf(stderr, "backswap: expected %d for %s\n", (want), (why)); \
|
||||
err = 1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
memset(&back, 0, sizeof(back));
|
||||
back.status = STATUS_READY;
|
||||
strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
|
||||
CHECK(1, "a plain ready slot");
|
||||
|
||||
back.tmpfile = back.tmpfile_buffer;
|
||||
strcpybuff(back.tmpfile_buffer, "/tmp/httrack-selftest.bin.bak");
|
||||
CHECK(0, "a slot still holding a re-fetch backup");
|
||||
|
||||
/* Callers clear a spent temporary by emptying the name, not the pointer. */
|
||||
back.tmpfile_buffer[0] = '\0';
|
||||
CHECK(1, "a slot whose temporary was already dropped");
|
||||
|
||||
back.tmpfile = NULL;
|
||||
back.locked = 1;
|
||||
CHECK(0, "a locked slot");
|
||||
back.locked = 0;
|
||||
|
||||
back.status = STATUS_TRANSFER;
|
||||
CHECK(0, "a slot still transferring");
|
||||
back.status = STATUS_READY;
|
||||
|
||||
back.url_sav[0] = '\0';
|
||||
CHECK(0, "a slot with no save name");
|
||||
|
||||
strcpybuff(back.url_sav, BACK_ADD_TEST);
|
||||
CHECK(0, "the dummy test slot");
|
||||
#undef CHECK
|
||||
|
||||
printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Put all backing entries that are ready in the storage hashtable to spare space and CPU */
|
||||
int back_cleanup_background(httrackp * opt, cache_back * cache,
|
||||
struct_back * sback) {
|
||||
@@ -585,12 +634,40 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Move src onto dst; RENAME does not clobber an existing target on Windows. */
|
||||
static hts_boolean replace_file(const char *src, const char *dst) {
|
||||
if (RENAME(src, dst) == 0)
|
||||
return HTS_TRUE;
|
||||
(void) UNLINK(dst);
|
||||
return RENAME(src, dst) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
/* Note: utf-8 */
|
||||
void back_refetch_backup(httrackp *opt, lien_back *const back) {
|
||||
back->tmpfile = NULL;
|
||||
if (fexist_utf8(back->url_sav)) {
|
||||
hts_boolean saved = HTS_FALSE;
|
||||
|
||||
if (create_back_tmpfile(opt, back, "bak") == 0) {
|
||||
/* clobber a .bak a killed run left behind, or the guard stays off for
|
||||
good (#758) */
|
||||
if (fexist_utf8(back->tmpfile))
|
||||
hts_log_print(opt, LOG_WARNING, "replacing leftover backup %s",
|
||||
back->tmpfile);
|
||||
saved = hts_rename_over(back->url_sav, back->tmpfile);
|
||||
}
|
||||
if (!saved) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"could not back up %s; an aborted re-fetch will lose it",
|
||||
back->url_sav);
|
||||
back->tmpfile = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Did the fetch fail to produce a response, as opposed to the engine
|
||||
deliberately passing the resource over? Only the latter may be purged. */
|
||||
static hts_boolean back_transfer_failed(const int statuscode) {
|
||||
switch (statuscode) {
|
||||
case STATUSCODE_TOO_BIG:
|
||||
case STATUSCODE_EXCLUDED:
|
||||
case STATUSCODE_TEST_OK:
|
||||
return HTS_FALSE;
|
||||
default:
|
||||
return statuscode <= 0 ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
|
||||
@@ -610,7 +687,7 @@ static void back_finalize_backup(httrackp *opt, lien_back *const back,
|
||||
}
|
||||
/* On failure keep the backup: an orphaned temp beats losing the good copy.
|
||||
*/
|
||||
if (!replace_file(back->tmpfile, back->url_sav))
|
||||
if (!hts_rename_over(back->tmpfile, back->url_sav))
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"could not restore %s; previous copy kept as %s",
|
||||
back->url_sav, back->tmpfile);
|
||||
@@ -741,7 +818,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
"Read error when decompressing");
|
||||
}
|
||||
UNLINK(unpacked);
|
||||
} else if (replace_file(unpacked, back[p].url_sav)) {
|
||||
} else if (hts_rename_over(unpacked, back[p].url_sav)) {
|
||||
/* The temp bypassed filecreate(), which is what chmods. */
|
||||
#ifndef _WIN32
|
||||
chmod(back[p].url_sav, HTS_ACCESS_FILE);
|
||||
@@ -1039,6 +1116,14 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
/* Aborted, error, or not ready: url_sav (if written) is broken; restore the
|
||||
previous copy from the backup. */
|
||||
back_finalize_backup(opt, &back[p], HTS_FALSE);
|
||||
/* Note the surviving copy, or the end-of-update purge drops what this run
|
||||
never managed to replace (#746). */
|
||||
if (!back[p].testmode && back_transfer_failed(back[p].r.statuscode) &&
|
||||
back[p].url_sav[0] != '\0' && fexist_utf8(back[p].url_sav)) {
|
||||
filenote(&opt->state.strc, back[p].url_sav, NULL);
|
||||
file_notify(opt, back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0,
|
||||
back[p].r.notmodified);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -3151,20 +3236,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].url_sav, 1, 1,
|
||||
back[i].r.notmodified);
|
||||
back[i].r.compressed = 0;
|
||||
/* Re-fetch over an existing file (#77 follow-up):
|
||||
move the good copy aside before truncating it
|
||||
so an aborted transfer can restore it. url_sav
|
||||
is still written normally (file list intact).
|
||||
*/
|
||||
back[i].tmpfile = NULL;
|
||||
if (fexist_utf8(back[i].url_sav)) {
|
||||
if (create_back_tmpfile(opt, &back[i], "bak") !=
|
||||
0 ||
|
||||
RENAME(back[i].url_sav, back[i].tmpfile) !=
|
||||
0) {
|
||||
back[i].tmpfile = NULL;
|
||||
}
|
||||
}
|
||||
back_refetch_backup(opt, &back[i]);
|
||||
if ((back[i].r.out =
|
||||
filecreate(&opt->state.strc,
|
||||
back[i].url_sav)) == NULL) {
|
||||
|
||||
@@ -139,6 +139,12 @@ int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p);
|
||||
int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p);
|
||||
/* Move the previous copy of back->url_sav to back->tmpfile so back_finalize()
|
||||
can put it back when the re-fetch fails (#77 follow-up). Call right before
|
||||
truncating url_sav; tmpfile stays NULL when there is nothing to save. */
|
||||
void back_refetch_backup(httrackp *opt, lien_back *const back);
|
||||
/* -#test=backswap: slots eligible for the on-disk ready table. */
|
||||
int back_selftest_slot_swap(void);
|
||||
void back_info(struct_back * sback, int i, int j, FILE * fp);
|
||||
void back_infostr(struct_back *sback, int i, int j, char *s, size_t size);
|
||||
LLint back_transferred(LLint add, struct_back * sback);
|
||||
|
||||
@@ -855,13 +855,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
return r;
|
||||
}
|
||||
|
||||
// lecture d'un fichier dans le cache
|
||||
// si save==null alors test unqiquement
|
||||
static int hts_rename(httrackp * opt, const char *a, const char *b) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: rename %s -> %s (%p %p)", a, b, a, b);
|
||||
return RENAME(a, b);
|
||||
}
|
||||
|
||||
/* Open the cache ZIP via hts_fopen_utf8 so a non-ASCII path_log isn't mangled
|
||||
to ANSI (#630); 64-bit funcs keep multi-GB caches whole on Windows LLP64. */
|
||||
static voidpf ZCALLBACK hts_zip_fopen_utf8(voidpf opaque, const void *filename,
|
||||
@@ -991,29 +984,15 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))) { // a previous cache exists.. rename it
|
||||
/* Remove OLD cache */
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")) !=
|
||||
0) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"Cache: error while moving previous cache");
|
||||
}
|
||||
}
|
||||
|
||||
/* Rename */
|
||||
if (hts_rename
|
||||
(opt,
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) != 0) {
|
||||
if (!hts_rename_over(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"),
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip"))) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"Cache: error while moving previous cache");
|
||||
} else {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: rotated new.zip to old.zip");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -676,7 +676,7 @@ int cache_selftests(httrackp *opt, const char *dir) {
|
||||
char base[HTS_URLMAXSIZE];
|
||||
|
||||
strcpybuff(base, dir);
|
||||
if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
|
||||
if (base[0] != '\0' && hts_lastchar(base) != '/') {
|
||||
strcatbuff(base, "/");
|
||||
}
|
||||
StringCopy(opt->path_log, base);
|
||||
@@ -856,7 +856,7 @@ static void golden_setup(httrackp *opt, const char *dir) {
|
||||
char base[HTS_URLMAXSIZE];
|
||||
|
||||
strcpybuff(base, dir);
|
||||
if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
|
||||
if (base[0] != '\0' && hts_lastchar(base) != '/') {
|
||||
strcatbuff(base, "/");
|
||||
}
|
||||
StringCopy(opt->path_log, base);
|
||||
|
||||
@@ -72,7 +72,7 @@ hts_codec hts_codec_parse(const char *encoding) {
|
||||
return HTS_CODEC_IDENTITY;
|
||||
if (strfield2(encoding, "gzip") || strfield2(encoding, "x-gzip") ||
|
||||
strfield2(encoding, "deflate") || strfield2(encoding, "x-deflate"))
|
||||
return HTS_USEZLIB ? HTS_CODEC_DEFLATE : HTS_CODEC_UNSUPPORTED;
|
||||
return HTS_CODEC_DEFLATE;
|
||||
if (strfield2(encoding, "br"))
|
||||
return HTS_USEBROTLI ? HTS_CODEC_BROTLI : HTS_CODEC_UNSUPPORTED;
|
||||
if (strfield2(encoding, "zstd"))
|
||||
@@ -98,16 +98,11 @@ hts_codec hts_codec_parse(const char *encoding) {
|
||||
const char *hts_acceptencoding(hts_boolean compressible, hts_boolean secure) {
|
||||
if (!compressible)
|
||||
return "identity";
|
||||
#if HTS_USEZLIB
|
||||
/* br and zstd over TLS only, as browsers do: a cleartext intermediary that
|
||||
rewrites a coding it can not read would corrupt the mirror. */
|
||||
if (secure)
|
||||
return "gzip, deflate" HTS_AE_BROTLI HTS_AE_ZSTD ", identity;q=0.9";
|
||||
return "gzip, deflate, identity;q=0.9";
|
||||
#else
|
||||
(void) secure;
|
||||
return "identity";
|
||||
#endif
|
||||
}
|
||||
|
||||
hts_boolean hts_codec_is_archive_ext(hts_codec codec, const char *ext) {
|
||||
@@ -300,11 +295,7 @@ int hts_codec_unpack(hts_codec codec, const char *filename,
|
||||
return -1;
|
||||
switch (codec) {
|
||||
case HTS_CODEC_DEFLATE:
|
||||
#if HTS_USEZLIB
|
||||
return hts_zunpack(filename, newfile);
|
||||
#else
|
||||
return -1;
|
||||
#endif
|
||||
case HTS_CODEC_BROTLI:
|
||||
case HTS_CODEC_ZSTD:
|
||||
break;
|
||||
@@ -348,10 +339,8 @@ size_t hts_codec_head(hts_codec codec, const void *in, size_t in_len, void *out,
|
||||
if (in == NULL || in_len == 0 || out == NULL || out_len == 0)
|
||||
return 0;
|
||||
switch (codec) {
|
||||
#if HTS_USEZLIB
|
||||
case HTS_CODEC_DEFLATE:
|
||||
return hts_zhead(in, in_len, out, out_len);
|
||||
#endif
|
||||
#if HTS_USEBROTLI
|
||||
case HTS_CODEC_BROTLI:
|
||||
return codec_head_brotli(in, in_len, out, out_len);
|
||||
|
||||
104
src/htscore.c
104
src/htscore.c
@@ -773,7 +773,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// sauter les + sans rien après..
|
||||
if (strnotempty(tempo)) {
|
||||
if ((plus == 0) && (type == 1)) { // implicite: *www.edf.fr par exemple
|
||||
if (tempo[strlen(tempo) - 1] != '*') {
|
||||
if (hts_lastchar(tempo) != '*') {
|
||||
strcatbuff(tempo, "*"); // ajouter un *
|
||||
}
|
||||
}
|
||||
@@ -1819,60 +1819,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// -- -- --
|
||||
// sauver fichier
|
||||
|
||||
/* En cas d'erreur, vérifier que fichier d'erreur existe */
|
||||
if (strnotempty(savename()) == 0) { // chemin de sauvegarde existant
|
||||
if (strcmp(urlfil(), "/robots.txt") == 0) { // pas robots.txt
|
||||
if (store_errpage) { // c'est une page d'erreur
|
||||
int create_html_warning = 0;
|
||||
int create_gif_warning = 0;
|
||||
|
||||
switch (ishtml(opt, urlfil())) { /* pas fichier html */
|
||||
case 0: /* non html */
|
||||
{
|
||||
char buff[256];
|
||||
|
||||
guess_httptype_sized(opt, buff, sizeof(buff), urlfil());
|
||||
if (strcmp(buff, "image/gif") == 0)
|
||||
create_gif_warning = 1;
|
||||
}
|
||||
break;
|
||||
case 1: /* html */
|
||||
if (!r.adr) {
|
||||
}
|
||||
break;
|
||||
default: /* don't know.. */
|
||||
break;
|
||||
}
|
||||
/* Créer message d'erreur ? */
|
||||
if (create_html_warning) {
|
||||
char *adr =
|
||||
(char *) malloct(strlen(HTS_DATA_ERROR_HTML) + 1100);
|
||||
hts_log_print(opt, LOG_INFO, "Creating HTML warning file (%s)",
|
||||
r.msg);
|
||||
if (adr) {
|
||||
if (r.adr) {
|
||||
freet(r.adr);
|
||||
r.adr = NULL;
|
||||
}
|
||||
sprintf(adr, HTS_DATA_ERROR_HTML, r.msg);
|
||||
r.adr = adr;
|
||||
}
|
||||
} else if (create_gif_warning) {
|
||||
char *adr = (char *) malloct(HTS_DATA_UNKNOWN_GIF_LEN);
|
||||
|
||||
hts_log_print(opt, LOG_INFO, "Creating GIF dummy file (%s)",
|
||||
r.msg);
|
||||
if (r.adr) {
|
||||
freet(r.adr);
|
||||
r.adr = NULL;
|
||||
}
|
||||
memcpy(adr, HTS_DATA_UNKNOWN_GIF, HTS_DATA_UNKNOWN_GIF_LEN);
|
||||
r.adr = adr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (strnotempty(savename()) == 0) { // pas de chemin de sauvegarde
|
||||
if (strcmp(urlfil(), "/robots.txt") == 0) { // robots.txt
|
||||
char BIGSTK sitemaps[8192];
|
||||
@@ -1980,10 +1926,9 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
|
||||
// ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
|
||||
// An empty body must not overwrite the file when the transfer failed
|
||||
// (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
|
||||
// copy to 0 (#77 follow-up).
|
||||
if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
|
||||
// A failed transfer has no body: r.adr holds debris from the aborted
|
||||
// read, which would destroy the copy being re-fetched (#748).
|
||||
if (r.statuscode > 0 && (r.adr != NULL || r.size == 0)) {
|
||||
file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
|
||||
if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
|
||||
0) {
|
||||
@@ -2191,7 +2136,9 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
continue;
|
||||
strcpybuff(file, StringBuff(opt->path_html));
|
||||
strcatbuff(file, line + 1);
|
||||
file[strlen(file) - 1] = '\0';
|
||||
/* strip filenote()'s ']', absent when linput() truncated the
|
||||
line */
|
||||
hts_striplastchar(file, ']');
|
||||
hts_changes_previous(opt, file + StringLength(opt->path_html));
|
||||
if (!strstr(adr, line)) { // not found in the new list?
|
||||
if (fexist_utf8(file)) { // still on disk
|
||||
@@ -2218,12 +2165,11 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
fseek(old_lst, 0, SEEK_SET);
|
||||
while(!feof(old_lst)) {
|
||||
linput(old_lst, line, 1000);
|
||||
while(strnotempty(line) && (line[strlen(line) - 1] != '/')
|
||||
&& (line[strlen(line) - 1] != '\\')) {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
while (strnotempty(line) && (hts_lastchar(line) != '/') &&
|
||||
(hts_lastchar(line) != '\\')) {
|
||||
hts_choplastchar(line);
|
||||
}
|
||||
if (strnotempty(line))
|
||||
line[strlen(line) - 1] = '\0';
|
||||
hts_choplastchar(line);
|
||||
if (strnotempty(line))
|
||||
if (!strstr(adr, line)) { // non trouvé?
|
||||
char BIGSTK file[HTS_URLMAXSIZE * 2];
|
||||
@@ -2236,13 +2182,12 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
if (opt->log) {
|
||||
hts_log_print(opt, LOG_INFO, "Purging directory %s/",
|
||||
file);
|
||||
while(strnotempty(file)
|
||||
&& (file[strlen(file) - 1] != '/')
|
||||
&& (file[strlen(file) - 1] != '\\')) {
|
||||
file[strlen(file) - 1] = '\0';
|
||||
while (strnotempty(file) &&
|
||||
(hts_lastchar(file) != '/') &&
|
||||
(hts_lastchar(file) != '\\')) {
|
||||
hts_choplastchar(file);
|
||||
}
|
||||
if (strnotempty(file))
|
||||
file[strlen(file) - 1] = '\0';
|
||||
hts_choplastchar(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2693,7 +2638,11 @@ HTSEXT_API int structcheck(const char *path) {
|
||||
if (!S_ISDIR(st.st_mode)) {
|
||||
#if HTS_REMOVE_ANNOYING_INDEX
|
||||
if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */
|
||||
sprintf(tmpbuf, "%s.txt", file);
|
||||
/* bounded here, not by the path-length guard far above */
|
||||
if (!sprintfbuff(tmpbuf, "%s.txt", file)) {
|
||||
errno = ENAMETOOLONG;
|
||||
return -1;
|
||||
}
|
||||
if (rename(file, tmpbuf) != 0) { /* Can't rename regular file */
|
||||
return -1;
|
||||
}
|
||||
@@ -2801,7 +2750,11 @@ HTSEXT_API int structcheck_utf8(const char *path) {
|
||||
if (!S_ISDIR(st.st_mode)) {
|
||||
#if HTS_REMOVE_ANNOYING_INDEX
|
||||
if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */
|
||||
sprintf(tmpbuf, "%s.txt", file);
|
||||
/* bounded here, not by the path-length guard far above */
|
||||
if (!sprintfbuff(tmpbuf, "%s.txt", file)) {
|
||||
errno = ENAMETOOLONG;
|
||||
return -1;
|
||||
}
|
||||
if (RENAME(file, tmpbuf) != 0) { /* Can't rename regular file */
|
||||
return -1;
|
||||
}
|
||||
@@ -3821,11 +3774,12 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
|
||||
strcpybuff(codebase, heap(ptr)->fil);
|
||||
else
|
||||
strcpybuff(codebase, heap(heap(ptr)->precedent)->fil);
|
||||
a = codebase + strlen(codebase) - 1;
|
||||
// empty codebase has no last char; codebase-1 would underflow
|
||||
a = codebase[0] != '\0' ? codebase + strlen(codebase) - 1 : codebase;
|
||||
while((*a) && (*a != '/') && (a > codebase))
|
||||
a--;
|
||||
if (*a == '/')
|
||||
*(a + 1) = '\0'; // couper
|
||||
*(a + 1) = '\0'; // cut
|
||||
} else { // couper http:// éventuel
|
||||
if (strfield(codebase, "http://")) {
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
|
||||
@@ -376,8 +376,8 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
|
||||
const char *template_footer, const char *adr,
|
||||
const char *fil);
|
||||
|
||||
// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
|
||||
// closed+NULLed on write. Precondition: ht_len>0.
|
||||
// Flush ht_buff[0..ht_len] to save on disk; *fp closed+NULLed on write.
|
||||
// Precondition: ht_len>0.
|
||||
void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
|
||||
FILE **fp, const char *ht_buff, size_t ht_len,
|
||||
const char *adr, const char *fil, const char *save);
|
||||
|
||||
@@ -357,7 +357,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char BIGSTK tempo[HTS_CDLMAXSIZE];
|
||||
|
||||
strcpybuff(tempo, argv[na] + 1);
|
||||
if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
|
||||
if (hts_lastchar(tempo) != '"') {
|
||||
char BIGSTK s[HTS_CDLMAXSIZE];
|
||||
|
||||
sprintf(s, "Missing quote in %s", argv[na]);
|
||||
@@ -365,7 +365,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
tempo[strlen(tempo) - 1] = '\0';
|
||||
hts_choplastchar(tempo);
|
||||
/* tempo is argv[na] minus its surrounding quotes, so it fits in place
|
||||
*/
|
||||
strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
|
||||
@@ -863,7 +863,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char BIGSTK tempo[HTS_CDLMAXSIZE + 256];
|
||||
|
||||
strcpybuff(tempo, argv[na] + 1);
|
||||
if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
|
||||
if (hts_lastchar(tempo) != '"') {
|
||||
char s[HTS_CDLMAXSIZE + 256];
|
||||
|
||||
sprintf(s, "Missing quote in %s", argv[na]);
|
||||
@@ -871,7 +871,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
tempo[strlen(tempo) - 1] = '\0';
|
||||
hts_choplastchar(tempo);
|
||||
/* tempo is argv[na] minus its surrounding quotes, so it fits in place
|
||||
*/
|
||||
strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
|
||||
@@ -2724,10 +2724,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char *a;
|
||||
|
||||
strcpybuff(rpath, StringBuff(opt->path_html));
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/')
|
||||
rpath[strlen(rpath) - 1] = '\0';
|
||||
}
|
||||
hts_striplastchar(rpath, '/');
|
||||
a = strrchr(rpath, '/');
|
||||
if (a) {
|
||||
*a = '\0';
|
||||
|
||||
@@ -624,6 +624,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
} else {
|
||||
file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1,
|
||||
0);
|
||||
/* Every failure exit below would else leave the mirror truncated
|
||||
(#771); the resume branch appends and needs no backup. */
|
||||
back_refetch_backup(opt, back);
|
||||
back->r.fp = filecreate(&opt->state.strc, back->url_sav);
|
||||
}
|
||||
strcpybuff(back->info, "receiving");
|
||||
|
||||
@@ -83,9 +83,8 @@ void infomsg(const char *msg) {
|
||||
/* try the flag as-is, then strip a trailing N as the numeric-arg
|
||||
placeholder (cN -> c); this order keeps -%N from becoming -% */
|
||||
p = optreal_find(cmd);
|
||||
if (p < 0 && (int) strlen(cmd) > 2 &&
|
||||
cmd[strlen(cmd) - 1] == 'N') {
|
||||
cmd[strlen(cmd) - 1] = '\0';
|
||||
if (p < 0 && (int) strlen(cmd) > 2 && hts_lastchar(cmd) == 'N') {
|
||||
hts_striplastchar(cmd, 'N');
|
||||
p = optreal_find(cmd);
|
||||
}
|
||||
if (p >= 0) {
|
||||
@@ -211,7 +210,7 @@ void help_wizard(httrackp * opt) {
|
||||
strcatbuff(str, "/websites/");
|
||||
}
|
||||
if (strnotempty(str))
|
||||
if ((str[strlen(str) - 1] != '/') && (str[strlen(str) - 1] != '\\'))
|
||||
if ((hts_lastchar(str) != '/') && (hts_lastchar(str) != '\\'))
|
||||
strcatbuff(str, "/");
|
||||
strcatbuff(stropt2, "-O \"");
|
||||
strcatbuff(stropt2, str);
|
||||
|
||||
16
src/htslib.c
16
src/htslib.c
@@ -1131,12 +1131,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
|
||||
// Compression accepted ?
|
||||
if (retour->req.http11) {
|
||||
hts_boolean compressible = HTS_FALSE;
|
||||
hts_boolean compressible =
|
||||
(!retour->req.range_used && !retour->req.nocompression);
|
||||
hts_boolean secure = HTS_FALSE;
|
||||
|
||||
#if HTS_USEZLIB
|
||||
compressible = (!retour->req.range_used && !retour->req.nocompression);
|
||||
#endif
|
||||
#if HTS_USEOPENSSL
|
||||
secure = retour->ssl ? HTS_TRUE : HTS_FALSE;
|
||||
#endif
|
||||
@@ -5080,7 +5078,7 @@ static int hts_dns_resolve_nocache_list(const char *const hostname,
|
||||
if (!strnotempty(hostname) || max <= 0) {
|
||||
return 0;
|
||||
}
|
||||
if ((hostname[0] == '[') && (hostname[strlen(hostname) - 1] == ']')) {
|
||||
if ((hostname[0] == '[') && (hts_lastchar(hostname) == ']')) {
|
||||
size_t size = strlen(hostname);
|
||||
char *copy = malloct(size + 1);
|
||||
int count;
|
||||
@@ -5493,9 +5491,8 @@ void cut_path(char *fullpath, char *path, size_t path_size, char *pname,
|
||||
size_t pname_size) {
|
||||
path[0] = pname[0] = '\0';
|
||||
if (strnotempty(fullpath)) {
|
||||
if ((fullpath[strlen(fullpath) - 1] == '/')
|
||||
|| (fullpath[strlen(fullpath) - 1] == '\\'))
|
||||
fullpath[strlen(fullpath) - 1] = '\0';
|
||||
if (!hts_striplastchar(fullpath, '/'))
|
||||
hts_striplastchar(fullpath, '\\');
|
||||
if (strlen(fullpath) > 1) {
|
||||
char *a;
|
||||
|
||||
@@ -6650,9 +6647,12 @@ int hts_rename_utf8(const char *oldpath, const char *newpath) {
|
||||
LPWSTR wnewpath = hts_pathToUCS2(newpath);
|
||||
if (woldpath != NULL && wnewpath != NULL) {
|
||||
const int result = _wrename(woldpath, wnewpath);
|
||||
/* Save errno: callers key off it (#779) and free() may clobber it. */
|
||||
const int err = errno;
|
||||
|
||||
free(woldpath);
|
||||
free(wnewpath);
|
||||
errno = err;
|
||||
return result;
|
||||
} else {
|
||||
if (woldpath != NULL)
|
||||
|
||||
@@ -43,9 +43,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsencoding.h"
|
||||
#include "htssniff.h"
|
||||
#include "htscodec.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
#include <ctype.h>
|
||||
#include <limits.h>
|
||||
|
||||
@@ -439,9 +437,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
strcpybuff(fil_complete_patche, normfil);
|
||||
// Version avec ou sans /
|
||||
if (fil_complete_patche[strlen(fil_complete_patche) - 1] == '/')
|
||||
fil_complete_patche[strlen(fil_complete_patche) - 1] = '\0';
|
||||
else
|
||||
if (!hts_striplastchar(fil_complete_patche, '/'))
|
||||
strcatbuff(fil_complete_patche, "/");
|
||||
i = hash_read(hash, normadr, fil_complete_patche, HASH_STRUCT_ORIGINAL_ADR_PATH); // recherche table 2 (former->adr+former->fil)
|
||||
if (i >= 0) {
|
||||
@@ -517,7 +513,8 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
&& protocol != PROTOCOL_FTP
|
||||
) {
|
||||
// tester type avec requète HEAD si on ne connait pas le type du fichier
|
||||
if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/'))) // slash doit être html?
|
||||
if (!((opt->check_type == 1) &&
|
||||
(hts_lastchar(fil) == '/'))) // slash doit être html?
|
||||
if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
|
||||
ishtml(opt, fil) < 0) { // unsure whether it's html or a file
|
||||
// lire dans le cache
|
||||
@@ -812,7 +809,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// - - - DEBUT NOMMAGE - - -
|
||||
|
||||
// Donner nom par défaut?
|
||||
if (fil[strlen(fil) - 1] == '/') {
|
||||
if (hts_lastchar(fil) == '/') {
|
||||
if (!strfield(adr_complete, "ftp://")
|
||||
) {
|
||||
strcatbuff(fil, DEFAULT_HTML); // nommer page par défaut!!
|
||||
@@ -1286,7 +1283,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
hts_lowcase(afs->save);
|
||||
|
||||
if (afs->save[strlen(afs->save) - 1] == '/')
|
||||
if (hts_lastchar(afs->save) == '/')
|
||||
strcatbuff(afs->save, DEFAULT_HTML); // nommer page par défaut!!
|
||||
}
|
||||
|
||||
|
||||
@@ -1701,7 +1701,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
#endif
|
||||
) // ok pas de problème
|
||||
url_ok = 1;
|
||||
else if (tempo[strlen(tempo) - 1] == '/') { // un slash: ok..
|
||||
else if (hts_lastchar(tempo) ==
|
||||
'/') { // un slash: ok..
|
||||
if (inscript) // sinon si pas javascript, méfiance (répertoire style base?)
|
||||
url_ok = 1;
|
||||
}
|
||||
@@ -2005,9 +2006,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (eadr - html - 1 < HTS_URLMAXSIZE) { // pas trop long?
|
||||
strncpy(lien, html, eadr - html - 1);
|
||||
lien[eadr - html - 1] = '\0';
|
||||
// supprimer les espaces
|
||||
while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
|
||||
lien[strlen(lien) - 1] = '\0';
|
||||
while (hts_striplastchar(lien, ' ')) {
|
||||
}
|
||||
|
||||
} else
|
||||
lien[0] = '\0'; // erreur
|
||||
@@ -2207,7 +2207,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// supposition dangereuse?
|
||||
// OUI!!
|
||||
#if HTS_TILDE_SLASH
|
||||
if (lien[strlen(lien) - 1] != '/') {
|
||||
if (hts_lastchar(lien) != '/') {
|
||||
char *a = lien + strlen(lien) - 1;
|
||||
|
||||
// éviter aussi index~1.html
|
||||
@@ -2272,7 +2272,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// Vérifier les codebase=applet (au lieu de applet/)
|
||||
if (p_type == -2) { // codebase
|
||||
if (strnotempty(lien)) {
|
||||
if (lien[strlen(lien) - 1] != '/') { // pas répertoire
|
||||
if (hts_lastchar(lien) != '/') { // pas répertoire
|
||||
strcatbuff(lien, "/");
|
||||
}
|
||||
}
|
||||
@@ -2688,9 +2688,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
int cat_data_len = 0;
|
||||
|
||||
// ajouter lien external
|
||||
switch ((link_has_authority(afs.af.adr)) ? 1
|
||||
: ((afs.af.fil[strlen(afs.af.fil) - 1] ==
|
||||
'/') ? 1 : (ishtml(opt, afs.af.fil)))) {
|
||||
switch ((link_has_authority(afs.af.adr))
|
||||
? 1
|
||||
: ((hts_lastchar(afs.af.fil) == '/')
|
||||
? 1
|
||||
: (ishtml(opt, afs.af.fil)))) {
|
||||
case 1:
|
||||
case -2: // html ou répertoire
|
||||
if (opt->getmode & HTS_GETMODE_HTML) {
|
||||
@@ -2733,7 +2735,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
cat_data_len = HTS_DATA_UNKNOWN_HTML_LEN;
|
||||
}
|
||||
break;
|
||||
} // html,gif
|
||||
} // html,gif
|
||||
|
||||
if (patch_it) {
|
||||
char BIGSTK save[HTS_URLMAXSIZE * 2];
|
||||
|
||||
@@ -539,6 +539,36 @@ static HTS_INLINE HTS_UNUSED HTS_PRINTF_FUN(3, 4) void slprintfbuff_clip(
|
||||
#define sprintfbuff(ARR, ...) slprintfbuff((ARR), sizeof(ARR), __VA_ARGS__)
|
||||
#endif
|
||||
|
||||
/* Last character of s, or '\0' when s is empty. Replaces s[strlen(s) - 1],
|
||||
which indexes one byte before the buffer on an empty string. */
|
||||
static HTS_INLINE HTS_UNUSED char hts_lastchar(const char *s) {
|
||||
const size_t len = strlen(s);
|
||||
|
||||
return len != 0 ? s[len - 1] : '\0';
|
||||
}
|
||||
|
||||
/* Drop a trailing c from s if present; HTS_TRUE if one was dropped. */
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean hts_striplastchar(char *s, char c) {
|
||||
const size_t len = strlen(s);
|
||||
|
||||
if (len != 0 && s[len - 1] == c) {
|
||||
s[len - 1] = '\0';
|
||||
return HTS_TRUE;
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Drop the last character of s whatever it is; HTS_TRUE if s was not empty. */
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean hts_choplastchar(char *s) {
|
||||
const size_t len = strlen(s);
|
||||
|
||||
if (len != 0) {
|
||||
s[len - 1] = '\0';
|
||||
return HTS_TRUE;
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
|
||||
added bounds checking. freet() also NULLs the freed pointer and tolerates
|
||||
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the
|
||||
|
||||
@@ -43,6 +43,7 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htsglobal.h"
|
||||
#include "htscore.h"
|
||||
#include "htsmodules.h"
|
||||
#include "htsback.h"
|
||||
#include "htsdefines.h"
|
||||
#include "htslib.h"
|
||||
@@ -63,9 +64,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htswarc.h"
|
||||
#include "htschanges.h"
|
||||
#include "htssinglefile.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
#if HTS_USEZSTD
|
||||
#include <zstd.h>
|
||||
#endif
|
||||
@@ -2623,6 +2622,68 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* an empty fil started htsAddLink's codebase walk before the buffer (#730) */
|
||||
static int st_addlink(httrackp *opt, int argc, char **argv) {
|
||||
htsmoduleStruct BIGSTK str;
|
||||
cache_back cache;
|
||||
struct_back *sback;
|
||||
hash_struct hash;
|
||||
int ptr = 0;
|
||||
int i;
|
||||
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.hashtable = (void *) coucal_new(0);
|
||||
sback = back_new(opt, opt->maxsoc * 32 + 1024);
|
||||
/* same wiring as hts_mirror (htscore.c) */
|
||||
hash_init(opt, &hash, opt->urlhack);
|
||||
hash.liens = (const lien_url *const *const *) &opt->liens;
|
||||
opt->hash = &hash;
|
||||
hts_record_init(opt);
|
||||
|
||||
memset(&str, 0, sizeof(str));
|
||||
str.opt = opt;
|
||||
str.sback = sback;
|
||||
str.cache = &cache;
|
||||
str.hashptr = &hash;
|
||||
str.ptr_ = &ptr;
|
||||
str.addLink = htsAddLink;
|
||||
|
||||
/* [0] is the underflow; [1] and [2] are controls that the trim is unchanged.
|
||||
A query-only link is the one that notices the trim at all: for the others
|
||||
ident_url_relatif() re-derives the directory from the path it is given. */
|
||||
for (i = 0; i < 3; i++) {
|
||||
static const char *const fil[3] = {"", "/dir/page.html", "/dir/page.html"};
|
||||
static const char *const lnk[3] = {"sub/page.html", "sub/page.html",
|
||||
"?x=1"};
|
||||
static const char *const want[3] = {
|
||||
"untouched", "http://www.example.com/dir/sub/page.html",
|
||||
"http://www.example.com/dir/?x=1"};
|
||||
char BIGSTK loc[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK link[HTS_URLMAXSIZE];
|
||||
|
||||
strcpybuff(loc, "untouched");
|
||||
strcpybuff(link, lnk[i]);
|
||||
str.localLink = loc;
|
||||
str.localLinkSize = (int) sizeof(loc);
|
||||
if (!hts_record_link(opt, "www.example.com", fil[i], "", "", "", ""))
|
||||
return 1;
|
||||
ptr = heap_top_index();
|
||||
str.url_host = heap(ptr)->adr;
|
||||
str.url_file = heap(ptr)->fil;
|
||||
assertf(htsAddLink(&str, link) == 0); /* refused by the wizard either way */
|
||||
if (strcmp(loc, want[i]) != 0) {
|
||||
fprintf(stderr, "addlink[%d]: got '%s' want '%s'\n", i, loc, want[i]);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
printf("addlink self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_cache(httrackp *opt, int argc, char **argv) {
|
||||
int err;
|
||||
|
||||
@@ -3264,6 +3325,81 @@ static int st_topindex(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Build a path of exactly len chars under base; returns that length. */
|
||||
static size_t st_structcheck_longpath(char *dst, size_t dstsize,
|
||||
const char *base, size_t len) {
|
||||
size_t n = strlen(base);
|
||||
|
||||
assertf(len < dstsize && n + 2 <= len);
|
||||
memmove(dst, base, n);
|
||||
while (n < len) {
|
||||
size_t seg = len - n - 1;
|
||||
|
||||
if (seg > 200) /* stay under the usual 255-byte component limit */
|
||||
seg = len - n == 202 ? 199 : 200; /* never leave a bare separator */
|
||||
dst[n++] = '/';
|
||||
memset(dst + n, 'x', seg);
|
||||
n += seg;
|
||||
}
|
||||
dst[n] = '\0';
|
||||
return n;
|
||||
}
|
||||
|
||||
/* The path guard, and the <name>.txt rename structcheck() performs when a
|
||||
regular file sits where a directory has to go (#745). */
|
||||
static int st_structcheck(httrackp *opt, int argc, char **argv) {
|
||||
char BIGSTK path[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK target[HTS_URLMAXSIZE * 2];
|
||||
FILE *fp;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "usage: -#test=structcheck <writable directory>\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* over the guard: refused before a single directory is created */
|
||||
st_structcheck_longpath(path, sizeof(path), argv[0], HTS_URLMAXSIZE + 1);
|
||||
errno = 0;
|
||||
assertf(structcheck(path) == -1);
|
||||
assertf(errno == EINVAL);
|
||||
errno = 0;
|
||||
assertf(structcheck_utf8(path) == -1);
|
||||
assertf(errno == EINVAL);
|
||||
{
|
||||
char *const sep = strchr(path + strlen(argv[0]) + 1, '/');
|
||||
|
||||
assertf(sep != NULL);
|
||||
sep[1] = '\0'; /* the outermost component it would have created */
|
||||
assertf(!dir_exists(path));
|
||||
}
|
||||
|
||||
/* a regular file where a directory belongs is renamed to <name>.txt */
|
||||
snprintf(path, sizeof(path), "%s/sc", argv[0]);
|
||||
fp = fopen(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
fclose(fp);
|
||||
snprintf(path, sizeof(path), "%s/sc/sub/", argv[0]);
|
||||
assertf(structcheck(path) == 0);
|
||||
assertf(dir_exists(path));
|
||||
snprintf(target, sizeof(target), "%s/sc.txt", argv[0]);
|
||||
assertf(fexist(target));
|
||||
|
||||
/* the utf-8 entry point carries the same rename */
|
||||
snprintf(path, sizeof(path), "%s/u8", argv[0]);
|
||||
fp = FOPEN(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
fclose(fp);
|
||||
snprintf(path, sizeof(path), "%s/u8/sub/", argv[0]);
|
||||
assertf(structcheck_utf8(path) == 0);
|
||||
assertf(dir_exists(path));
|
||||
snprintf(target, sizeof(target), "%s/u8.txt", argv[0]);
|
||||
assertf(fexist_utf8(target));
|
||||
|
||||
printf("structcheck self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Each inplace_escape_*() must equal escape_*() on a copy. */
|
||||
static int st_inplace_escape(httrackp *opt, int argc, char **argv) {
|
||||
/* >255 bytes forces the helper's malloct path, not the stack buffer */
|
||||
@@ -3377,7 +3513,6 @@ static int st_status(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if HTS_USEZLIB
|
||||
/* Deflate src->path at windowBits (16+ gzip, + zlib, - raw); 0 on success. */
|
||||
static int ae_write_packed(const char *path, int windowBits,
|
||||
const unsigned char *src, size_t len) {
|
||||
@@ -3451,7 +3586,6 @@ static int ae_write_collision(const char *path, const unsigned char *src,
|
||||
freet(buf);
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Write src[0..len) to path as-is; 0 on success. */
|
||||
static int ae_write_raw(const char *path, const unsigned char *src,
|
||||
@@ -3501,7 +3635,6 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
|
||||
assertf(strstr(on, "br") == NULL && strstr(on, "zstd") == NULL);
|
||||
assertf((strstr(tls, ", br") != NULL) == (HTS_USEBROTLI != 0));
|
||||
assertf((strstr(tls, "zstd") != NULL) == (HTS_USEZSTD != 0));
|
||||
#if HTS_USEZLIB
|
||||
if (argc >= 1) {
|
||||
static const int windowBits[] = {16 + MAX_WBITS, MAX_WBITS, -MAX_WBITS};
|
||||
const unsigned char small[] =
|
||||
@@ -3580,10 +3713,6 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
|
||||
}
|
||||
freet(body);
|
||||
}
|
||||
#else
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
#endif
|
||||
printf("acceptencoding self-test OK: %s\n", on);
|
||||
return 0;
|
||||
}
|
||||
@@ -3928,7 +4057,6 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
|
||||
freet(big);
|
||||
}
|
||||
|
||||
#if HTS_USEZLIB
|
||||
/* A highly compressible document decodes without running away: the ratio
|
||||
budget cannot bind (deflate tops out near 1032:1), so this pins the
|
||||
decompression path itself rather than the 64 MiB ceiling. */
|
||||
@@ -3979,13 +4107,11 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
|
||||
freet(z);
|
||||
freet(x);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* An unterminated <loc> at end of buffer must not read past it. */
|
||||
assertf(sm_scan("<urlset><loc>http://h.test/a", 100, &idx, &c) == 0);
|
||||
assertf(sm_scan("<urlset><lo", 100, &idx, &c) == 0);
|
||||
|
||||
#if HTS_USEZLIB
|
||||
/* A gzip-framed document is decompressed before scanning. */
|
||||
{
|
||||
const char *const xml =
|
||||
@@ -4015,7 +4141,6 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
|
||||
assertf(hts_sitemap_scan(z, 4, 100, &idx, sm_take, &c) == -1);
|
||||
freet(z);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* robots.txt: only Sitemap: records, comments stripped, case-insensitive,
|
||||
and group-independent (no User-agent line needed). */
|
||||
@@ -5830,6 +5955,104 @@ static int st_mirrorio(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void ro_put(const char *path, const char *data) {
|
||||
FILE *const fp = FOPEN(path, "wb");
|
||||
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, strlen(data), fp) == strlen(data));
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* HTS_TRUE if path holds exactly data. */
|
||||
static hts_boolean ro_is(const char *path, const char *data) {
|
||||
char buf[64];
|
||||
FILE *const fp = FOPEN(path, "rb");
|
||||
size_t n;
|
||||
|
||||
if (fp == NULL)
|
||||
return HTS_FALSE;
|
||||
n = fread(buf, 1, sizeof(buf), fp);
|
||||
fclose(fp);
|
||||
return n == strlen(data) && memcmp(buf, data, n) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
// -#test=renameover <dir>: hts_rename_over() must replace an existing dst and
|
||||
// never delete one it did not replace (#779). Which half is live depends on
|
||||
// what rename() does to an existing target, so probe that and name the regime.
|
||||
static int st_renameover(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "renameover: needs a writable base dir\n");
|
||||
return 1;
|
||||
}
|
||||
char src[HTS_URLMAXSIZE * 2], dst[HTS_URLMAXSIZE * 2];
|
||||
int err = 0;
|
||||
|
||||
fconcat(src, sizeof(src), argv[0], "renameover-src.bin");
|
||||
fconcat(dst, sizeof(dst), argv[0], "renameover-dst.bin");
|
||||
|
||||
(void) UNLINK(src);
|
||||
(void) UNLINK(dst);
|
||||
ro_put(src, "probe");
|
||||
ro_put(dst, "probe");
|
||||
|
||||
const int probe = RENAME(src, dst) == 0 ? 0 : errno;
|
||||
/* Only a target in the way is something the unlink can clear. */
|
||||
const hts_boolean replaceable = probe == 0 || probe == EEXIST;
|
||||
|
||||
printf("renameover: regime %s\n",
|
||||
probe == 0 ? "clobber" : (probe == EEXIST ? "fallback" : "refused"));
|
||||
|
||||
(void) UNLINK(src);
|
||||
(void) UNLINK(dst);
|
||||
ro_put(src, "new");
|
||||
ro_put(dst, "old");
|
||||
if (replaceable) {
|
||||
/* An existing dst must still be replaced: the unlink is for this. */
|
||||
if (!hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: replacing an existing dst failed: %s\n",
|
||||
strerror(errno));
|
||||
err++;
|
||||
} else if (!ro_is(dst, "new") || fexist_utf8(src)) {
|
||||
fprintf(stderr, "renameover: dst was not replaced by src\n");
|
||||
err++;
|
||||
}
|
||||
} else {
|
||||
/* A failure the unlink cannot fix must leave dst as it was. */
|
||||
if (hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: an unfixable failure reported success\n");
|
||||
err++;
|
||||
}
|
||||
if (!ro_is(dst, "old")) {
|
||||
fprintf(stderr, "renameover: an unfixable failure destroyed dst\n");
|
||||
err++;
|
||||
}
|
||||
}
|
||||
|
||||
/* A missing src must leave dst alone and report failure. */
|
||||
(void) UNLINK(src);
|
||||
ro_put(dst, "keep");
|
||||
if (hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: a missing src reported success\n");
|
||||
err++;
|
||||
}
|
||||
if (!ro_is(dst, "keep")) {
|
||||
fprintf(stderr, "renameover: a missing src destroyed dst\n");
|
||||
err++;
|
||||
}
|
||||
|
||||
/* Same, with dst absent too: nothing to lose, still a failure. */
|
||||
(void) UNLINK(dst);
|
||||
if (hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: a missing src and dst reported success\n");
|
||||
err++;
|
||||
}
|
||||
|
||||
(void) UNLINK(dst);
|
||||
printf("renameover: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
// -#test=direnum <dir>: enumerate a long+non-ASCII directory via the
|
||||
// opendir/readdir wrappers; children must round-trip as UTF-8 (#133,#630).
|
||||
static int st_direnum(httrackp *opt, int argc, char **argv) {
|
||||
@@ -6089,6 +6312,123 @@ static int st_changes(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/* #747: a thread is outstanding from the moment hts_newthread() returns, not
|
||||
from the moment it starts running, or a wait right after the spawn joins
|
||||
nothing. One thread per round is what makes the old bug visible: the wait
|
||||
had to find the counter at zero, and a batch of spawns gives the earlier
|
||||
threads time to raise it. Unfixed, one round in two caught it, so the round
|
||||
count is what turns that into a reliable failure. */
|
||||
#define THREADWAIT_N 8
|
||||
#define THREADWAIT_ROUNDS 16
|
||||
#define THREADWAIT_SLEEP_MS 50
|
||||
#define THREADWAIT_GATE_MS 10000
|
||||
|
||||
static htsmutex threadwait_lock = HTSMUTEX_INIT;
|
||||
static int threadwait_done = 0;
|
||||
static hts_boolean threadwait_gated = HTS_FALSE;
|
||||
|
||||
static int threadwait_count(void) {
|
||||
int n;
|
||||
|
||||
hts_mutexlock(&threadwait_lock);
|
||||
n = threadwait_done;
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
return n;
|
||||
}
|
||||
|
||||
static void threadwait_thread(void *arg) {
|
||||
(void) arg;
|
||||
Sleep(THREADWAIT_SLEEP_MS);
|
||||
hts_mutexlock(&threadwait_lock);
|
||||
threadwait_done++;
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
}
|
||||
|
||||
/* Stays outstanding until the gate clears, so wait_n() can be asked to leave a
|
||||
known number of live threads behind. Bounded: a wait_n() that wrongly drains
|
||||
them would otherwise never return, and hang the suite instead of failing. */
|
||||
static void threadwait_gated_thread(void *arg) {
|
||||
int waited;
|
||||
|
||||
(void) arg;
|
||||
for (waited = 0; waited < THREADWAIT_GATE_MS; waited += 10) {
|
||||
hts_boolean gated;
|
||||
|
||||
hts_mutexlock(&threadwait_lock);
|
||||
gated = threadwait_gated;
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
if (!gated)
|
||||
break;
|
||||
Sleep(10);
|
||||
}
|
||||
hts_mutexlock(&threadwait_lock);
|
||||
threadwait_done++;
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
}
|
||||
|
||||
static int st_backswap(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
return back_selftest_slot_swap();
|
||||
}
|
||||
|
||||
static int st_threadwait(httrackp *opt, int argc, char **argv) {
|
||||
int err = 0;
|
||||
int i, round;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
/* htsthread_wait() joins a thread spawned just before it */
|
||||
for (round = 0; round < THREADWAIT_ROUNDS && !err; round++) {
|
||||
hts_mutexlock(&threadwait_lock);
|
||||
threadwait_done = 0;
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
if (hts_newthread(threadwait_thread, NULL) != 0) {
|
||||
fprintf(stderr, "threadwait: cannot spawn\n");
|
||||
return 1;
|
||||
}
|
||||
htsthread_wait();
|
||||
if (threadwait_count() != 1) {
|
||||
fprintf(stderr, "threadwait: round %d returned before the thread ran\n",
|
||||
round);
|
||||
err = 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* htsthread_wait_n(n) leaves n behind rather than draining everything */
|
||||
hts_mutexlock(&threadwait_lock);
|
||||
threadwait_done = 0;
|
||||
threadwait_gated = HTS_TRUE;
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
for (i = 0; i < THREADWAIT_N; i++) {
|
||||
if (hts_newthread(threadwait_gated_thread, NULL) != 0) {
|
||||
fprintf(stderr, "threadwait: cannot spawn a gated thread\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
htsthread_wait_n(THREADWAIT_N);
|
||||
if (threadwait_count() != 0) {
|
||||
fprintf(stderr, "threadwait: wait_n(%d) joined %d gated threads\n",
|
||||
THREADWAIT_N, threadwait_count());
|
||||
err = 1;
|
||||
}
|
||||
hts_mutexlock(&threadwait_lock);
|
||||
threadwait_gated = HTS_FALSE;
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
htsthread_wait();
|
||||
if (threadwait_count() != THREADWAIT_N) {
|
||||
fprintf(stderr, "threadwait: wait left %d/%d gated threads running\n",
|
||||
THREADWAIT_N - threadwait_count(), THREADWAIT_N);
|
||||
err = 1;
|
||||
}
|
||||
|
||||
printf("threadwait self-test: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
#define CHANGES_RACE_FILES 8
|
||||
#define CHANGES_RACE_ROUNDS 400
|
||||
|
||||
@@ -6103,11 +6443,8 @@ static void changes_race_notify(httrackp *opt, int n) {
|
||||
hts_changes_notify(opt, "race.example", fil, save, HTS_TRUE, HTS_FALSE);
|
||||
}
|
||||
|
||||
/* htsthread_wait() counts a thread only once it is running, so it can return
|
||||
before any of them started; join on our own counter instead. */
|
||||
static htsmutex changes_race_lock = HTSMUTEX_INIT;
|
||||
static int changes_race_started = 0;
|
||||
static int changes_race_live = 0;
|
||||
|
||||
static int changes_race_count(int *which) {
|
||||
int n;
|
||||
@@ -6127,9 +6464,6 @@ static void changes_race_thread(void *arg) {
|
||||
hts_mutexrelease(&changes_race_lock);
|
||||
for (i = 0; i < CHANGES_RACE_ROUNDS; i++)
|
||||
changes_race_notify(opt, i % CHANGES_RACE_FILES);
|
||||
hts_mutexlock(&changes_race_lock);
|
||||
changes_race_live--;
|
||||
hts_mutexrelease(&changes_race_lock);
|
||||
}
|
||||
|
||||
/* A transfer thread the crawl never joins (FTP) reaches hts_changes_notify()
|
||||
@@ -6145,7 +6479,7 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
|
||||
return 1;
|
||||
}
|
||||
strcpybuff(base, argv[0]);
|
||||
if (base[0] != '\0' && base[strlen(base) - 1] != '/')
|
||||
if (base[0] != '\0' && hts_lastchar(base) != '/')
|
||||
strcatbuff(base, "/");
|
||||
StringCopy(opt->path_html, base);
|
||||
StringCopy(opt->path_html_utf8, base);
|
||||
@@ -6184,7 +6518,6 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
|
||||
threads reaching a fresh one together race on the init itself. */
|
||||
hts_mutexlock(&changes_race_lock);
|
||||
changes_race_started = 0;
|
||||
changes_race_live = 4;
|
||||
hts_mutexrelease(&changes_race_lock);
|
||||
for (i = 0; i < 4; i++) {
|
||||
if (hts_newthread(changes_race_thread, opt) != 0) {
|
||||
@@ -6198,8 +6531,7 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
|
||||
for (i = 0; i < 64; i++)
|
||||
hts_changes_report(opt, &out);
|
||||
hts_changes_close_opt(opt);
|
||||
while (changes_race_count(&changes_race_live) > 0)
|
||||
Sleep(10);
|
||||
htsthread_wait();
|
||||
|
||||
/* Sealed: a straggler must be dropped, not start a report nobody writes. */
|
||||
changes_race_notify(opt, CHANGES_RACE_FILES + 1);
|
||||
@@ -6218,6 +6550,88 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/* The x[strlen(x) - 1] class (#770). The string starts mid-arena so the byte
|
||||
it must not touch is a real neighbour; poisoned with '#', not 0, or a stray
|
||||
NUL terminator would read as untouched. */
|
||||
static int st_lastchar(httrackp *opt, int argc, char **argv) {
|
||||
enum { off = 8 };
|
||||
|
||||
char arena[16];
|
||||
char *const s = &arena[off];
|
||||
const int guard = off - 1; /* what the old idiom clobbers */
|
||||
int err = 0;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
#define REPOISON(str) \
|
||||
do { \
|
||||
memset(arena, '#', sizeof(arena)); \
|
||||
strlcpybuff(s, (str), sizeof(arena) - off); \
|
||||
} while (0)
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
printf(" FAIL line %d: %s\n", __LINE__, #cond); \
|
||||
err = 1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* the empty string: every helper must report "nothing" and touch nothing */
|
||||
REPOISON("");
|
||||
CHECK(hts_lastchar(s) == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
REPOISON("");
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
|
||||
CHECK(arena[guard] == '#');
|
||||
CHECK(s[0] == '\0');
|
||||
REPOISON("");
|
||||
CHECK(hts_choplastchar(s) == HTS_FALSE);
|
||||
CHECK(arena[guard] == '#');
|
||||
CHECK(s[0] == '\0');
|
||||
|
||||
/* a '/' sitting where the underflow would land must not be mistaken for the
|
||||
string's own last byte -- this is the #768 shape */
|
||||
REPOISON("");
|
||||
arena[guard] = '/';
|
||||
CHECK(hts_lastchar(s) == '\0');
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
|
||||
CHECK(arena[guard] == '/');
|
||||
|
||||
/* non-empty: ordinary behaviour */
|
||||
REPOISON("ab/");
|
||||
CHECK(hts_lastchar(s) == '/');
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_TRUE);
|
||||
CHECK(strcmp(s, "ab") == 0);
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
|
||||
CHECK(strcmp(s, "ab") == 0);
|
||||
CHECK(hts_choplastchar(s) == HTS_TRUE);
|
||||
CHECK(strcmp(s, "a") == 0);
|
||||
CHECK(hts_choplastchar(s) == HTS_TRUE);
|
||||
CHECK(s[0] == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
/* one-character string: the boundary the guards get wrong */
|
||||
REPOISON("/");
|
||||
CHECK(hts_lastchar(s) == '/');
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_TRUE);
|
||||
CHECK(s[0] == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
/* control: the canary must be able to fail, or the checks above prove
|
||||
nothing. Clobber it exactly as the unguarded idiom would. */
|
||||
REPOISON("");
|
||||
s[-1] = '\0';
|
||||
CHECK(arena[guard] != '#');
|
||||
|
||||
#undef REPOISON
|
||||
#undef CHECK
|
||||
|
||||
printf("lastchar self-test: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* Registry: name -> handler, with a usage hint and a one-line description. */
|
||||
/* ------------------------------------------------------------ */
|
||||
@@ -6273,10 +6687,16 @@ static const struct selftest_entry {
|
||||
{"strsafe", "[overflow|overflow-buff|overflow-src [str]]",
|
||||
"bounded string-op self-test", st_strsafe},
|
||||
{"copyopt", "", "copy_htsopt option-copy self-test", st_copyopt},
|
||||
{"lastchar", "", "last-char helpers never index before the buffer (#770)",
|
||||
st_lastchar},
|
||||
{"changes", "", "--changes bucket accounting and JSON escaping (#714)",
|
||||
st_changes},
|
||||
{"changes-race", "<dir>", "--changes under a late transfer thread (#714)",
|
||||
st_changes_race},
|
||||
{"threadwait", "", "htsthread_wait() joins threads spawned just before it",
|
||||
st_threadwait},
|
||||
{"backswap", "", "which backlog slots may be swapped to the ready table",
|
||||
st_backswap},
|
||||
{"pause", "", "randomized inter-file pause target self-test", st_pause},
|
||||
{"relative", "<link> <curr-file>", "relative link between two paths",
|
||||
st_relative},
|
||||
@@ -6307,6 +6727,8 @@ static const struct selftest_entry {
|
||||
{"fsize", "<dir>", "file size past the 2GB signed-32-bit wrap", st_fsize},
|
||||
{"growsize", "", "buffer capacity for a 64-bit file size (no int wrap)",
|
||||
st_growsize},
|
||||
{"addlink", "", "htsAddLink codebase walk over an empty current path",
|
||||
st_addlink},
|
||||
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
|
||||
{"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
|
||||
st_cacheindex},
|
||||
@@ -6330,6 +6752,9 @@ static const struct selftest_entry {
|
||||
{"useragent", "", "default User-Agent self-test", st_useragent},
|
||||
{"makeindex", "[dir]", "hts_finish_makeindex footer/refresh self-test",
|
||||
st_makeindex},
|
||||
{"structcheck", "<dir>",
|
||||
"structcheck path guard and the <name>.txt rename it performs",
|
||||
st_structcheck},
|
||||
{"topindex", "[dir]",
|
||||
"hts_buildtopindex charset handling of a non-ASCII project dir",
|
||||
st_topindex},
|
||||
@@ -6369,6 +6794,10 @@ static const struct selftest_entry {
|
||||
{"mirrorio", "<dir>",
|
||||
"round-trip a long+non-ASCII path through the mirror I/O wrappers",
|
||||
st_mirrorio},
|
||||
{"renameover", "<dir>",
|
||||
"hts_rename_over(): replace dst, but never delete a dst it did not "
|
||||
"replace",
|
||||
st_renameover},
|
||||
{"direnum", "<dir>",
|
||||
"enumerate a long+non-ASCII directory through opendir/readdir",
|
||||
st_direnum},
|
||||
|
||||
@@ -1050,9 +1050,9 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
if (!linput(fp, line, sizeof(line) - 2)) {
|
||||
*str = '\0';
|
||||
}
|
||||
if (*str && str[strlen(str) - 1] == '\\') {
|
||||
if (hts_lastchar(str) == '\\') {
|
||||
nocr = 1;
|
||||
str[strlen(str) - 1] = '\0';
|
||||
hts_striplastchar(str, '\\');
|
||||
}
|
||||
while(*str) {
|
||||
char *pos;
|
||||
@@ -1169,11 +1169,8 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
char *rpath = (char *) adr;
|
||||
|
||||
//find_handle h;
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/') {
|
||||
rpath[strlen(rpath) - 1] = '\0'; /* note: patching stored (inhash) value */
|
||||
}
|
||||
}
|
||||
/* note: patching stored (inhash) value */
|
||||
hts_striplastchar(rpath, '/');
|
||||
{
|
||||
const char *profiles = hts_getcategories(rpath, 0);
|
||||
const char *categ = hts_getcategories(rpath, 1);
|
||||
|
||||
@@ -1108,13 +1108,8 @@ hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
|
||||
(void) chmod(fconv(catbuff, sizeof(catbuff), StringBuff(tmp)),
|
||||
HTS_ACCESS_FILE);
|
||||
#endif
|
||||
if (ok) {
|
||||
/* RENAME does not clobber an existing target on Windows. */
|
||||
if (RENAME(StringBuff(tmp), page_path) != 0) {
|
||||
(void) UNLINK(page_path);
|
||||
ok = RENAME(StringBuff(tmp), page_path) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
}
|
||||
if (ok)
|
||||
ok = hts_rename_over(StringBuff(tmp), page_path);
|
||||
if (!ok) {
|
||||
hts_log_print(opt, LOG_ERROR, "single-file: could not rewrite %s",
|
||||
page_path);
|
||||
|
||||
@@ -985,10 +985,7 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
|
||||
&& toptemplate_bodycat) {
|
||||
|
||||
strcpybuff(rpath, path);
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/')
|
||||
rpath[strlen(rpath) - 1] = '\0';
|
||||
}
|
||||
hts_striplastchar(rpath, '/');
|
||||
|
||||
fpo = fopen(fconcat(catbuff, sizeof(catbuff), rpath, "/index.html"), "wb");
|
||||
if (fpo) {
|
||||
@@ -1201,11 +1198,8 @@ HTSEXT_API char *hts_getcategories(char *path, int type) {
|
||||
find_handle h;
|
||||
coucal hashCateg = NULL;
|
||||
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/') {
|
||||
rpath[strlen(rpath) - 1] = '\0'; /* note: patching stored (inhash) value */
|
||||
}
|
||||
}
|
||||
/* note: patching stored (inhash) value */
|
||||
hts_striplastchar(rpath, '/');
|
||||
h = hts_findfirst(rpath);
|
||||
if (h) {
|
||||
String iname = STRING_EMPTY;
|
||||
@@ -1302,7 +1296,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
|
||||
|
||||
strcpybuff(rpath, path);
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] != '\\')
|
||||
if (hts_lastchar(rpath) != '\\')
|
||||
strcatbuff(rpath, "\\");
|
||||
}
|
||||
strcatbuff(rpath, "*.*");
|
||||
@@ -1314,7 +1308,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
|
||||
strcpybuff(find->path, path);
|
||||
{
|
||||
if (find->path[0]) {
|
||||
if (find->path[strlen(find->path) - 1] != '/')
|
||||
if (hts_lastchar(find->path) != '/')
|
||||
strcatbuff(find->path, "/");
|
||||
}
|
||||
}
|
||||
@@ -1443,3 +1437,22 @@ HTSEXT_API hts_boolean hts_findissystem(find_handle find) {
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
hts_boolean hts_rename_over(const char *src, const char *dst) {
|
||||
char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
|
||||
|
||||
fconv(csrc, sizeof(csrc), src);
|
||||
fconv(cdst, sizeof(cdst), dst);
|
||||
if (RENAME(csrc, cdst) == 0)
|
||||
return HTS_TRUE;
|
||||
/* Only a dst in the way is something the unlink can clear, and the CRT maps
|
||||
that to EEXIST; it keeps EACCES for a src another process holds, where
|
||||
removing dst would lose a file the retry cannot replace (#790). The src
|
||||
check covers a CRT that reports neither. */
|
||||
const int err = errno;
|
||||
|
||||
if (err != EEXIST || !fexist_utf8(src))
|
||||
return HTS_FALSE;
|
||||
(void) UNLINK(cdst);
|
||||
return RENAME(csrc, cdst) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
@@ -137,6 +137,12 @@ HTSEXT_API hts_boolean hts_findisdir(find_handle find);
|
||||
HTSEXT_API hts_boolean hts_findisfile(find_handle find);
|
||||
HTSEXT_API hts_boolean hts_findissystem(find_handle find);
|
||||
|
||||
/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both
|
||||
paths are fconv()'d. dst is removed only to make room for a src that exists,
|
||||
so a caller whose src was never written keeps its dst; a retry that still
|
||||
fails does not (#790). */
|
||||
hts_boolean hts_rename_over(const char *src, const char *dst);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
147
src/htswarc.c
147
src/htswarc.c
@@ -57,6 +57,9 @@ Please visit our Website: http://www.httrack.com
|
||||
/* opt->state.warc value meaning "open failed once, do not retry". */
|
||||
#define WARC_DISABLED ((void *) ~(uintptr_t) 0)
|
||||
|
||||
/* Suffix of the in-progress archive when a previous one must survive it. */
|
||||
#define WARC_TMP_SUFFIX ".tmp"
|
||||
|
||||
struct warc_writer {
|
||||
FILE *f;
|
||||
httrackp *opt; /* kept for close-time logging (warc_wacz_package) */
|
||||
@@ -86,6 +89,11 @@ struct warc_writer {
|
||||
char *base_path; /* resolved archive path minus .warc[.gz] suffix */
|
||||
const char *base_ext; /* ".warc.gz" or ".warc" (static) */
|
||||
char *arc_path; /* full single-file archive path (NULL under rotation) */
|
||||
/* A re-run must not destroy an archive it cannot replace (#759). */
|
||||
hts_boolean protect_prev; /* previous archive present: build in a temp */
|
||||
hts_boolean opened; /* open completed; a failed one swaps nothing */
|
||||
hts_boolean failed; /* a record or segment was lost: swap nothing */
|
||||
uint64_t unbacked_revisits; /* revisits whose payload no file here holds */
|
||||
char **page_lines; /* one JSON page line per 200 text/html response, owned */
|
||||
size_t page_count;
|
||||
size_t page_cap;
|
||||
@@ -964,16 +972,6 @@ static zipFile wacz_zip_open(const char *path) {
|
||||
return zipOpen2_64(path, 0 /*create*/, NULL, &ff);
|
||||
}
|
||||
|
||||
/* Move src onto dst (UTF-8/Windows-safe); RENAME won't clobber on Windows, so
|
||||
fall back to unlink+rename. Returns 0 on success. */
|
||||
static int wacz_rename_over(const char *src, const char *dst) {
|
||||
char cs[CATBUFF_SIZE], cd[CATBUFF_SIZE];
|
||||
if (RENAME(fconv(cs, sizeof(cs), src), fconv(cd, sizeof(cd), dst)) == 0)
|
||||
return 0;
|
||||
(void) UNLINK(fconv(cd, sizeof(cd), dst));
|
||||
return RENAME(fconv(cs, sizeof(cs), src), fconv(cd, sizeof(cd), dst));
|
||||
}
|
||||
|
||||
/* Package the segment(s) + .cdx + a generated pages.jsonl into <base>.wacz at
|
||||
crawl end (the archive file(s) and .cdx are already closed on disk). */
|
||||
static void warc_wacz_package(warc_writer *w) {
|
||||
@@ -1092,7 +1090,7 @@ static void warc_wacz_package(warc_writer *w) {
|
||||
hts_log_print(w->opt, LOG_WARNING,
|
||||
"WACZ: packaging failed, kept existing %s untouched",
|
||||
waczpath);
|
||||
} else if (wacz_rename_over(tmppath, waczpath) != 0) {
|
||||
} else if (!hts_rename_over(tmppath, waczpath)) {
|
||||
(void) UNLINK(fconv(catbuff, sizeof(catbuff), tmppath));
|
||||
hts_log_print(w->opt, LOG_WARNING | LOG_ERRNO,
|
||||
"WACZ: could not finalize %s", waczpath);
|
||||
@@ -1139,13 +1137,17 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
|
||||
never split a record, and never rotate a warcinfo (it opens a segment). */
|
||||
if (w->max_size > 0 && w->seg_base != NULL && w->offset >= w->max_size &&
|
||||
strcmp(type, "warcinfo") != 0) {
|
||||
if (warc_rotate(w) != 0)
|
||||
if (warc_rotate(w) != 0) {
|
||||
w->failed = HTS_TRUE;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* F4: overflow-safe block length; http_hdr_len+sep is provably small. */
|
||||
if (payload > (size_t) -1 - http_hdr_len - sep)
|
||||
if (payload > (size_t) -1 - http_hdr_len - sep) {
|
||||
w->failed = HTS_TRUE;
|
||||
return -1;
|
||||
}
|
||||
block_len = http_hdr_len + sep + payload;
|
||||
|
||||
memset(&hdr, 0, sizeof(hdr));
|
||||
@@ -1251,11 +1253,23 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
|
||||
rc = 0;
|
||||
done:
|
||||
wbuf_free(&hdr);
|
||||
if (rc != 0)
|
||||
w->failed = HTS_TRUE; /* a truncated run must not replace a whole one */
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ---- segment rotation (--warc-max-size) ---- */
|
||||
|
||||
/* Path to open for the segment whose final path is `final`: that path itself,
|
||||
or a sibling temp while a previous archive must survive until close. */
|
||||
static const char *warc_open_path(warc_writer *w, const char *final, char *buf,
|
||||
size_t bufsz) {
|
||||
if (!w->protect_prev)
|
||||
return final;
|
||||
snprintf(buf, bufsz, "%s" WARC_TMP_SUFFIX, final);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Emit the warcinfo that heads a segment; sets w->info_id for its records. */
|
||||
static int warc_write_warcinfo_record(warc_writer *w) {
|
||||
w->info_id[0] = '\0'; /* warcinfo itself carries no WARC-Warcinfo-ID */
|
||||
@@ -1267,17 +1281,23 @@ static int warc_write_warcinfo_record(warc_writer *w) {
|
||||
|
||||
static int warc_rotate(warc_writer *w) {
|
||||
char namebuf[HTS_URLMAXSIZE * 2];
|
||||
char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
if (w->f != NULL) {
|
||||
fclose(w->f);
|
||||
w->f = NULL;
|
||||
}
|
||||
w->seg++;
|
||||
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
|
||||
const unsigned next = w->seg + 1;
|
||||
FILE *f;
|
||||
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, next,
|
||||
w->seg_ext);
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), namebuf), "wb");
|
||||
if (w->f == NULL)
|
||||
/* Open before advancing: w->seg must only ever name a segment that exists,
|
||||
or close-time packaging and swapping would work on a missing file. */
|
||||
f = FOPEN(fconv(catbuff, sizeof(catbuff),
|
||||
warc_open_path(w, namebuf, openbuf, sizeof(openbuf))),
|
||||
"wb");
|
||||
if (f == NULL)
|
||||
return -1;
|
||||
if (w->f != NULL)
|
||||
fclose(w->f);
|
||||
w->f = f;
|
||||
w->seg = next;
|
||||
w->offset = 0;
|
||||
if (w->cdx_on) {
|
||||
freet(w->cur_seg);
|
||||
@@ -1334,6 +1354,7 @@ void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path) {
|
||||
warc_writer *warc_open(httrackp *opt, const char *path) {
|
||||
warc_writer *w;
|
||||
char namebuf[HTS_URLMAXSIZE * 2];
|
||||
char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
wbuf info;
|
||||
const char *robots;
|
||||
@@ -1484,35 +1505,105 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
|
||||
path = namebuf;
|
||||
}
|
||||
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
|
||||
if (w->max_size == 0 && (w->arc_path = strdupt(path)) == NULL) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
/* Set only once arc_path is recorded, so a half-built writer never swaps. */
|
||||
w->protect_prev = fsize_utf8(path) > 0 ? HTS_TRUE : HTS_FALSE;
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff),
|
||||
warc_open_path(w, path, openbuf, sizeof(openbuf))),
|
||||
"wb");
|
||||
if (w->f == NULL) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
if (w->cdx_on)
|
||||
w->cur_seg = path_basename_dup(path);
|
||||
if (w->wacz_on && w->max_size == 0)
|
||||
w->arc_path = strdupt(path); /* single-file: package this exact path */
|
||||
|
||||
if (warc_write_warcinfo_record(w) != 0) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
w->opened = HTS_TRUE;
|
||||
return w;
|
||||
}
|
||||
|
||||
/* Final path of segment s (the run's only archive when rotation is off). */
|
||||
static const char *warc_seg_path(warc_writer *w, unsigned s, char *buf,
|
||||
size_t bufsz) {
|
||||
if (w->max_size == 0)
|
||||
return w->arc_path;
|
||||
snprintf(buf, bufsz, "%s-%05u%s", w->seg_base, s, w->seg_ext);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Swap this run's archive into place, unless it only holds revisits naming
|
||||
bodies the previous archive still has and this one does not (#759).
|
||||
HTS_FALSE: the previous archive was kept, so leave its .cdx and .wacz too. */
|
||||
static hts_boolean warc_commit(warc_writer *w) {
|
||||
char finalbuf[HTS_URLMAXSIZE * 2];
|
||||
char tmpbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const unsigned nseg = (w->max_size > 0) ? w->seg + 1 : 1;
|
||||
hts_boolean swap;
|
||||
unsigned s;
|
||||
|
||||
if (!w->protect_prev)
|
||||
return HTS_TRUE; /* nothing was there to lose: written in place */
|
||||
|
||||
/* All or nothing: a swap stopping halfway would mix this run's segments with
|
||||
the previous one's, so require every temp before renaming any. */
|
||||
swap = w->opened && !w->failed && w->unbacked_revisits == 0;
|
||||
for (s = 0; s < nseg && swap; s++) {
|
||||
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
|
||||
warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
|
||||
swap = fsize_utf8(tmpbuf) > 0;
|
||||
}
|
||||
|
||||
if (swap) {
|
||||
for (s = 0; s < nseg; s++) {
|
||||
const char *final = warc_seg_path(w, s, finalbuf, sizeof(finalbuf));
|
||||
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX, final);
|
||||
if (!hts_rename_over(tmpbuf, final)) {
|
||||
hts_log_print(w->opt, LOG_ERROR | LOG_ERRNO,
|
||||
"WARC: could not replace %s", final);
|
||||
return HTS_FALSE;
|
||||
}
|
||||
}
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
for (s = 0; s < nseg; s++) {
|
||||
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
|
||||
warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
|
||||
(void) UNLINK(fconv(catbuff, sizeof(catbuff), tmpbuf));
|
||||
}
|
||||
if (w->unbacked_revisits > 0)
|
||||
hts_log_print(
|
||||
w->opt, LOG_ERROR,
|
||||
"WARC: this pass revisited %llu URL(s) without re-downloading them, so "
|
||||
"its archive would name bodies no file holds; kept the previous %s "
|
||||
"(re-run with -C0, or --warc-file with a name of its own)",
|
||||
(unsigned long long) w->unbacked_revisits,
|
||||
warc_seg_path(w, 0, finalbuf, sizeof(finalbuf)));
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
void warc_close(warc_writer *w) {
|
||||
size_t i;
|
||||
if (w == NULL)
|
||||
return;
|
||||
warc_cdx_flush(w); /* sort + write <base>.cdx before tearing down */
|
||||
if (w->f != NULL)
|
||||
fclose(w->f);
|
||||
w->f = NULL;
|
||||
if (warc_commit(w)) {
|
||||
warc_cdx_flush(w); /* sort + write <base>.cdx beside the archive */
|
||||
#if HTS_USEOPENSSL
|
||||
if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
|
||||
warc_wacz_package(w);
|
||||
if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
|
||||
warc_wacz_package(w);
|
||||
#endif
|
||||
}
|
||||
if (w->seen != NULL)
|
||||
coucal_delete(&w->seen);
|
||||
for (i = 0; i < w->cdx_count; i++)
|
||||
@@ -1593,6 +1684,8 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
|
||||
if (is_update_unchanged) {
|
||||
is_revisit = 1;
|
||||
profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
|
||||
/* Served from cache: the payload sits in the previous archive, not here. */
|
||||
w->unbacked_revisits++;
|
||||
} else if (have_pdig && w->seen != NULL) {
|
||||
void *prev = NULL;
|
||||
if (coucal_read_pvoid(w->seen, pdig, &prev) && prev != NULL) {
|
||||
|
||||
21
src/htsweb.c
21
src/htsweb.c
@@ -101,8 +101,8 @@ static void htsweb_sig_brpipe(int code) {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
/* Number of background threads */
|
||||
static int background_threads = 0;
|
||||
/* Threads that never return; no wait may count on them draining. */
|
||||
static int nonjoinable_threads = 0;
|
||||
|
||||
/* Server/client ping handling */
|
||||
static htsmutex pingMutex = HTSMUTEX_INIT;
|
||||
@@ -224,9 +224,7 @@ int main(int argc, char *argv[]) {
|
||||
#ifdef HTS_USESWF
|
||||
smallserver_setkey("USESWF", "1");
|
||||
#endif
|
||||
#ifdef HTS_USEZLIB
|
||||
smallserver_setkey("USEZLIB", "1");
|
||||
#endif
|
||||
#ifdef _WIN32
|
||||
smallserver_setkey("WIN32", "1");
|
||||
#endif
|
||||
@@ -299,15 +297,19 @@ int main(int argc, char *argv[]) {
|
||||
|
||||
/* pinger */
|
||||
if (parentPid > 0) {
|
||||
hts_newthread(client_ping, (void *) (uintptr_t) parentPid);
|
||||
background_threads++; /* Do not wait for this thread! */
|
||||
if (hts_newthread(client_ping, (void *) (uintptr_t) parentPid) == 0) {
|
||||
#ifndef _WIN32
|
||||
nonjoinable_threads++; /* client_ping() only ever leaves through exit() */
|
||||
#endif
|
||||
}
|
||||
smallserver_setpinghandler(pingHandler, NULL);
|
||||
}
|
||||
|
||||
/* launch */
|
||||
ret = help_server(argv[1], defaultPort, bindAddr);
|
||||
|
||||
htsthread_wait_n(background_threads - 1);
|
||||
/* Drain everything a mirror may still have in flight, the pinger aside. */
|
||||
htsthread_wait_n(nonjoinable_threads);
|
||||
hts_uninit();
|
||||
|
||||
#ifdef _WIN32
|
||||
@@ -382,7 +384,6 @@ void webhttrack_main(char *cmd) {
|
||||
commandRunning = 1;
|
||||
DEBUG(fprintf(stderr, "commandRunning=1\n"));
|
||||
hts_newthread(back_launch_cmd, (void *) strdup(cmd));
|
||||
background_threads++; /* Do not wait for this thread! */
|
||||
}
|
||||
|
||||
void webhttrack_lock(void) {
|
||||
@@ -423,8 +424,8 @@ static int webhttrack_runmain(httrackp * opt, int argc, char **argv) {
|
||||
/* Rock'in! */
|
||||
ret = hts_main2(argc, argv, opt);
|
||||
|
||||
/* Wait for pending threads to finish */
|
||||
htsthread_wait_n(background_threads);
|
||||
/* Wait for pending threads to finish; the pinger and this thread stay. */
|
||||
htsthread_wait_n(nonjoinable_threads + 1);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -40,7 +40,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscodec.h"
|
||||
#include "htszlib.h"
|
||||
|
||||
#if HTS_USEZLIB
|
||||
/* zlib */
|
||||
/*
|
||||
#include <zlib.h>
|
||||
@@ -274,4 +273,3 @@ const char *hts_get_zerror(int err) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
7
tests/01_engine-addlink.test
Normal file
7
tests/01_engine-addlink.test
Normal file
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# an empty current path underflowed htsAddLink's codebase walk (#730).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
httrack -O /dev/null -#test=addlink | grep -q "addlink self-test OK"
|
||||
8
tests/01_engine-backswap.test
Normal file
8
tests/01_engine-backswap.test
Normal file
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# A ready slot still owning a temporary must stay in memory: swapping it out
|
||||
# clears the entry, which unlinks the re-fetch backup (#771).
|
||||
httrack -O /dev/null -#test=backswap | grep -q "backswap self-test: OK"
|
||||
35
tests/01_engine-lastchar.test
Normal file
35
tests/01_engine-lastchar.test
Normal file
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Runs the lastchar self-test (#770), then keeps the idiom it replaced from
|
||||
# coming back: x[strlen(x) - 1] indexes one byte before the buffer when x is
|
||||
# empty, and the guard is never where the index is.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
out=$(httrack -#test=lastchar) || {
|
||||
echo "httrack -#test=lastchar exited non-zero: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Exact-match the success line so a renamed/removed test can't pass silently.
|
||||
test "$out" = "lastchar self-test: OK" || {
|
||||
echo "expected 'lastchar self-test: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The self-test only covers the helpers, so guard the class at the source: the
|
||||
# lone survivor is inside a comment block in htsname.c. coucal is vendored, so
|
||||
# its idioms are not ours to police.
|
||||
hits=$(command grep -rn 'strlen([^)]*) *- *1\]' "$top_srcdir/src" \
|
||||
--include='*.c' --include='*.h' --exclude-dir=coucal |
|
||||
grep -v 'htsname\.c:.*save\[strlen(save)-1\]' |
|
||||
grep -v 'Replaces s\[strlen(s) - 1\]' |
|
||||
grep -v 'The x\[strlen(x) - 1\] class' || true)
|
||||
|
||||
test -z "$hits" || {
|
||||
echo "x[strlen(x) - 1] reintroduced; use hts_lastchar/hts_striplastchar/hts_choplastchar:" >&2
|
||||
echo "$hits" >&2
|
||||
exit 1
|
||||
}
|
||||
55
tests/01_engine-renameover.test
Normal file
55
tests/01_engine-renameover.test
Normal file
@@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Drives -#test=renameover: hts_rename_over() must replace an existing dst, and
|
||||
# must leave dst alone when the rename failed for a reason removing dst cannot
|
||||
# fix (#779). The selftest prints the regime it detected; pin it per platform so
|
||||
# a leg cannot pass having tested the other half.
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
case "$(uname -s)" in
|
||||
MINGW* | MSYS_NT*) want=fallback ;; # native rename() refuses an existing target
|
||||
*) want=clobber ;;
|
||||
esac
|
||||
|
||||
out=$(httrack -O /dev/null -#test=renameover "$dir")
|
||||
echo "$out" | grep -q "renameover: OK"
|
||||
echo "$out" | grep -q "renameover: regime $want"
|
||||
|
||||
if [ "$(uname -s)" != "Linux" ]; then
|
||||
echo "renameover: LD_PRELOAD interposition is Linux-only here, skipping"
|
||||
exit 0
|
||||
fi
|
||||
# A --disable-shared build has nothing to preload; anything else missing is a
|
||||
# build problem, not a skip, or the interposed legs would pass vacuously.
|
||||
if [ ! -r "${RENAMEFAIL_LA:-}" ]; then
|
||||
echo "renameover: ${RENAMEFAIL_LA:-\$RENAMEFAIL_LA} was not built" >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q "^dlname=''" "$RENAMEFAIL_LA"; then
|
||||
echo "renameover: static-only build, skipping the interposed legs"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -r "${RENAMEFAIL_LIB:-}" ]; then
|
||||
echo "renameover: ${RENAMEFAIL_LIB:-\$RENAMEFAIL_LIB} was not built" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# An LD_PRELOAD library loads ahead of the executable's own libasan, which ASan
|
||||
# refuses by default. The shim allocates nothing, so the ordering is harmless.
|
||||
export ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}verify_asan_link_order=0"
|
||||
|
||||
# The unlink fallback is dead code on POSIX, so borrow Windows' rename().
|
||||
out=$(LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
|
||||
-#test=renameover "$dir")
|
||||
echo "$out" | grep -q "renameover: OK"
|
||||
echo "$out" | grep -q "renameover: regime fallback"
|
||||
|
||||
# A source another process holds fails with EACCES, which dst had no part in.
|
||||
out=$(RENAMEFAIL_MODE=locked LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
|
||||
-#test=renameover "$dir")
|
||||
echo "$out" | grep -q "renameover: OK"
|
||||
echo "$out" | grep -q "renameover: regime refused"
|
||||
12
tests/01_engine-structcheck.test
Normal file
12
tests/01_engine-structcheck.test
Normal file
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# structcheck(): the path-length guard, and the <name>.txt rename that once
|
||||
# built its target with an unbounded sprintf (#745).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=structcheck "$dir" |
|
||||
grep -q "structcheck self-test OK"
|
||||
28
tests/01_engine-threadwait.test
Normal file
28
tests/01_engine-threadwait.test
Normal file
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_threadwait_st.XXXXXX") || exit 1
|
||||
trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# No pipe into grep: SIGPIPE would mask a failing exit status.
|
||||
expect_ok() {
|
||||
local label="$1" out
|
||||
shift
|
||||
out=$("$@" 2>&1) || {
|
||||
echo "FAIL: ${label} exited non-zero: ${out}"
|
||||
exit 1
|
||||
}
|
||||
case "$out" in
|
||||
*"${label}: OK"*) ;;
|
||||
*)
|
||||
echo "FAIL: ${out}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# A thread is outstanding from the moment hts_newthread() returns, so a wait
|
||||
# that follows the spawn joins it, and wait_n(n) still leaves n behind (#747).
|
||||
expect_ok "threadwait self-test" httrack -O "${tmpdir}/o1" -#test=threadwait
|
||||
111
tests/100_local-purge-longpath.test
Normal file
111
tests/100_local-purge-longpath.test
Normal file
@@ -0,0 +1,111 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The end-of-mirror purge read old.lst with linput(fp, line, 1000), which caps a
|
||||
# chunk at 999 bytes. filenote() writes "[<save path>]", so a 998-byte save path
|
||||
# makes a 1000-byte line that comes back as 999 bytes plus a one-byte "]" tail.
|
||||
# For that tail `line + 1` is empty, and with no -O so is path_html, so the purge
|
||||
# indexed file[-1] (#770). Needs no -O, hence not local-crawl.sh.
|
||||
#
|
||||
# The OOB write lands in stack padding, so only the sanitized CI leg fails on
|
||||
# unfixed code; the length assertion below is what keeps this from passing
|
||||
# vacuously if the save path ever stops landing on the chunk boundary.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
# python3 is required; mirror check-network.sh's skip-with-77 convention.
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
# Resolve httrack now: the harness prepends a RELATIVE dir to PATH, so the cd
|
||||
# below would otherwise silently fall through to an installed copy.
|
||||
httrack_bin=$(command -v httrack)
|
||||
test -n "$httrack_bin" || {
|
||||
echo "httrack not on PATH" >&2
|
||||
exit 1
|
||||
}
|
||||
httrack_bin=$(cd "$(dirname "$httrack_bin")" && pwd)/$(basename "$httrack_bin")
|
||||
|
||||
work=$(mktemp -d)
|
||||
trap 'kill -9 "${spid:-}" 2>/dev/null || true; rm -rf "$work"' EXIT
|
||||
|
||||
mkdir -p "$work/root" "$work/proj"
|
||||
|
||||
"$python" "$top_srcdir/tests/local-server.py" --root "$work/root" \
|
||||
>"$work/server.out" 2>"$work/server.err" &
|
||||
spid=$!
|
||||
|
||||
port=
|
||||
for _ in $(seq 1 100); do
|
||||
port=$(sed -n 's/^PORT \([0-9]*\)$/\1/p' "$work/server.out" 2>/dev/null || true)
|
||||
test -n "$port" && break
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "local-server.py did not announce a port" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Save path is "127.0.0.1_<port>/" + the URL path; target 998 bytes total.
|
||||
prefix="127.0.0.1_${port}/"
|
||||
need=$((998 - ${#prefix}))
|
||||
|
||||
# macOS caps PATH_MAX at 1024, so neither the docroot nor the mirror can hold a
|
||||
# 998-byte path, and httrack's own HTS_MAX_PATH_LEN clamps below it too. The
|
||||
# trigger is simply not reachable there.
|
||||
pathmax=$(getconf PATH_MAX / 2>/dev/null || echo 4096)
|
||||
test "$pathmax" -ge $((need + ${#work} + 128)) || {
|
||||
echo "PATH_MAX $pathmax too small for a $need-byte path; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
# Same on Win32, where MAX_PATH clamps the save path; getconf answers for the
|
||||
# MSYS emulation, not the Win32 API httrack calls, so it cannot see this.
|
||||
case "$(uname -s)" in
|
||||
MINGW* | MSYS* | CYGWIN*)
|
||||
echo "Win32 MAX_PATH cannot hold a $need-byte save path; skipping" >&2
|
||||
exit 77
|
||||
;;
|
||||
esac
|
||||
|
||||
urlpath=""
|
||||
while test $((need - ${#urlpath})) -gt 60; do
|
||||
urlpath="${urlpath}$(printf 'd%02d' ${#urlpath})$(printf 'a%.0s' $(seq 1 46))/"
|
||||
done
|
||||
urlpath="${urlpath}$(printf 'f%.0s' $(seq 1 $((need - ${#urlpath} - 5)))).html"
|
||||
test ${#urlpath} -eq "$need"
|
||||
|
||||
mkdir -p "$work/root/$(dirname "$urlpath")"
|
||||
printf '<html><body>leaf</body></html>\n' >"$work/root/$urlpath"
|
||||
printf '<html><body><a href="/%s">x</a></body></html>\n' "$urlpath" \
|
||||
>"$work/root/index.html"
|
||||
|
||||
cd "$work/proj" # no -O: opt->path_html stays empty
|
||||
|
||||
"$httrack_bin" "http://127.0.0.1:$port/" -q -%v0 -s0 -r3 >"$work/crawl1.log" 2>&1 || true
|
||||
|
||||
lst=$work/proj/hts-cache/new.lst
|
||||
test -f "$lst" || {
|
||||
echo "pass 1 produced no new.lst" >&2
|
||||
exit 1
|
||||
}
|
||||
# Arms the test: without a line of exactly 1000 bytes linput() never splits and
|
||||
# the purge never sees a one-byte tail.
|
||||
awk 'length == 1000 { found = 1 } END { exit !found }' "$lst" || {
|
||||
echo "no 1000-byte line in new.lst; the chunk split is not exercised" >&2
|
||||
awk '{ print length }' "$lst" | sort -n | tail -3 >&2
|
||||
echo "--- urlpath len ${#urlpath}, prefix $prefix" >&2
|
||||
tail -20 "$work/crawl1.log" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Pass 2 renames new.lst to old.lst and runs it through the purge loop.
|
||||
"$httrack_bin" "http://127.0.0.1:$port/" -q -%v0 -s0 -r3 >"$work/crawl2.log" 2>&1
|
||||
|
||||
test -s "$work/proj/$prefix$urlpath" || {
|
||||
echo "the mirrored leaf did not survive the purge" >&2
|
||||
exit 1
|
||||
}
|
||||
27
tests/101_local-update-stale-bak.test
Normal file
27
tests/101_local-update-stale-bak.test
Normal file
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A .bak left behind by a killed run must not disable the #77 re-fetch backup
|
||||
# (#758): a leftover file (slow.bin.bak) has to be clobbered so the -M abort can
|
||||
# still restore the pass-1 copy, and a leftover the engine cannot clobber
|
||||
# (fast.bin.bak, planted as a directory) has to be reported instead of silently
|
||||
# truncating with no safety net. Only the Windows leg arms the first half:
|
||||
# POSIX rename() clobbers on its own.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--plant-file bigtrunc/slow.bin.bak \
|
||||
--plant-dir bigtrunc/fast.bin.bak \
|
||||
--rerun-args '--update -M400000' \
|
||||
--log-found 'More than 400000 bytes have been transferred.. giving up' \
|
||||
--log-found 'replacing leftover backup .*slow\.bin\.bak' \
|
||||
--log-found 'could not back up .*fast\.bin' \
|
||||
--log-not-found 'could not back up .*slow\.bin' \
|
||||
--log-not-found 'could not restore' \
|
||||
--found bigtrunc/slow.bin \
|
||||
--found bigtrunc/fast.bin \
|
||||
--file-min-bytes bigtrunc/slow.bin 655360 \
|
||||
--file-not-matches bigtrunc/slow.bin 'stale-leftover' \
|
||||
httrack 'BASEURL/bigtrunc/index.html' -c2
|
||||
163
tests/102_local-ftp-refetch.test
Normal file
163
tests/102_local-ftp-refetch.test
Normal file
@@ -0,0 +1,163 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An FTP re-fetch used to truncate the mirror before the transfer, so a failed
|
||||
# one destroyed the previous copy (#771). keep.bin is cut short and empty.bin
|
||||
# gets nothing; both must keep their old bytes. stay.bin takes the same path and
|
||||
# completes, so a fix that merely stopped writing would fail here.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
command -v httrack >/dev/null || {
|
||||
echo "could not find httrack" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
server=$(nativepath "${testdir}/ftp-server.py")
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
|
||||
serverpid=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
root="${tmpdir}/root"
|
||||
out="${tmpdir}/crawl"
|
||||
mode="${tmpdir}/mode"
|
||||
report="${out}/hts-changes.json"
|
||||
mkdir -p "$root" "$out"
|
||||
|
||||
# The two generations differ in length, or a re-fetch that resumed at EOF
|
||||
# instead of truncating would leave the same bytes and pass for the wrong reason.
|
||||
write_bodies() {
|
||||
local gen="$1" fill="$2" name
|
||||
for name in keep empty stay; do
|
||||
"$python" -c 'import sys; sys.stdout.buffer.write(
|
||||
("%s-FTP-%s " % (sys.argv[1].upper(), sys.argv[2])).encode()
|
||||
+ sys.argv[2][-1].encode() * int(sys.argv[3]))' "$name" "$gen" "$fill" \
|
||||
>"${root}/${name}.bin"
|
||||
done
|
||||
}
|
||||
write_bodies V1 4096
|
||||
: >"$mode"
|
||||
|
||||
serverlog="${tmpdir}/server.out"
|
||||
"$python" "$server" --root "$(nativepath "$root")" \
|
||||
--mode-file "$(nativepath "$mode")" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 300); do
|
||||
line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "ftp server exited early: $(cat "$serverlog")" >&2
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover ftp server port: $(cat "$serverlog")" >&2
|
||||
exit 1
|
||||
}
|
||||
host="127.0.0.1_${port}"
|
||||
urls=()
|
||||
for name in keep empty stay; do
|
||||
urls+=("ftp://127.0.0.1:${port}/${name}.bin")
|
||||
done
|
||||
# -c1: a parallel FTP crawl loses whole transfers to a separate, older bug in
|
||||
# the backlog slot swap (#797), which would flake this test on a loaded runner.
|
||||
common=(--quiet --disable-security-limits --robots=0 --timeout=20 --max-time=120
|
||||
--retries=1 -c1 --changes)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
test ! -f "$report" || cat "$report" >&2
|
||||
exit 1
|
||||
}
|
||||
ok() { echo "OK: $*"; }
|
||||
size_of() { wc -c <"$1" | tr -d '[:space:]'; }
|
||||
# Files listed under a report bucket, one "file:size" per line.
|
||||
listed() {
|
||||
"$python" - "$report" "$1" <<'EOF' | tr -d '\r'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as fp:
|
||||
report = json.load(fp)
|
||||
for entry in report[sys.argv[2]]:
|
||||
print("%s:%s" % (entry["file"], entry.get("size", "none")))
|
||||
EOF
|
||||
}
|
||||
|
||||
# --- pass 1: a healthy mirror ------------------------------------------------
|
||||
# Bounded like every local-crawl.sh pass: a wedged crawl must fail the test, not
|
||||
# hang the job until CI cancels it and discards the log (#796).
|
||||
run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" \
|
||||
>"${tmpdir}/log1" 2>&1
|
||||
for name in keep empty stay; do
|
||||
test -s "${out}/${host}/${name}.bin" || fail "pass 1 did not mirror ${name}.bin"
|
||||
done
|
||||
mkdir "${tmpdir}/snap"
|
||||
cp "${out}/${host}"/*.bin "${tmpdir}/snap/"
|
||||
ok "pass 1 mirrored three files"
|
||||
|
||||
# --- pass 2: the same three re-fetched, two of them failing -------------------
|
||||
# norest throughout: an accepted REST sends the client down the append path,
|
||||
# which is not the one that truncates.
|
||||
write_bodies V2 6000
|
||||
cat >"$mode" <<'EOF'
|
||||
/keep.bin truncate norest
|
||||
/empty.bin empty norest
|
||||
/stay.bin norest
|
||||
EOF
|
||||
run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" --update \
|
||||
>"${tmpdir}/log2" 2>&1
|
||||
|
||||
# Both failures have to be the transfer dying mid-body: a connect or lookup
|
||||
# failure never opens the file, and everything below would pass vacuously.
|
||||
for name in keep empty; do
|
||||
grep -aqE "Error:.*\"FTP file incomplete\".*${name}\.bin" "${out}/hts-log.txt" ||
|
||||
fail "${name}.bin did not fail during the transfer"
|
||||
done
|
||||
ok "both re-fetches reached the body and failed there"
|
||||
|
||||
for name in keep empty; do
|
||||
cmp -s "${out}/${host}/${name}.bin" "${tmpdir}/snap/${name}.bin" ||
|
||||
fail "${name}.bin differs from the copy pass 1 left ($(size_of \
|
||||
"${out}/${host}/${name}.bin") bytes now)"
|
||||
done
|
||||
ok "a failed FTP transfer left the previously mirrored bytes alone"
|
||||
|
||||
grep -aq "FTP-V2 " "${out}/${host}/stay.bin" ||
|
||||
fail "stay.bin was not refreshed; the re-fetch writes nothing at all now"
|
||||
ok "a successful FTP re-fetch still replaces the file"
|
||||
|
||||
leftover=$(find "${out}/${host}" -name '*.bak')
|
||||
test -z "$leftover" || fail "backup left behind: ${leftover}"
|
||||
ok "no backup temporary survived the pass"
|
||||
|
||||
# --purge-old is on by default: a kept file the run never replaced must still be
|
||||
# in new.lst, or it is deleted as gone (#746).
|
||||
for name in keep empty stay; do
|
||||
test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was purged"
|
||||
done
|
||||
ok "the kept copies survived the update purge"
|
||||
|
||||
# --- the change report is the mirror's own account of the pass ----------------
|
||||
test -s "$report" || fail "pass 2 wrote no ${report}"
|
||||
test -z "$(listed gone)" || fail "the report calls something gone: $(listed gone)"
|
||||
for name in keep empty; do
|
||||
listed unchanged | grep -qx "${host}/${name}.bin:$(size_of "${tmpdir}/snap/${name}.bin")" ||
|
||||
fail "${name}.bin is not reported unchanged at its previous size"
|
||||
if listed changed | grep -q "^${host}/${name}.bin:"; then
|
||||
fail "${name}.bin is reported changed as well"
|
||||
fi
|
||||
done
|
||||
listed changed | grep -qx "${host}/stay.bin:$(size_of "${out}/${host}/stay.bin")" ||
|
||||
fail "stay.bin is not reported changed"
|
||||
ok "the change report calls the kept files unchanged and the refreshed one changed"
|
||||
@@ -1,12 +1,14 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An update run against a dead server must not destroy the cache: the no-data
|
||||
# rollback restores the previous hts-cache generation (zip caches lost it).
|
||||
# rollback restores the previous hts-cache generation (zip caches lost it). The
|
||||
# live update pass leaves an old.zip behind, so the dead pass rotates onto an
|
||||
# existing generation instead of into empty space.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun-dead \
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun --rerun-dead \
|
||||
--found 'simple/basic.html' \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
|
||||
@@ -18,6 +18,7 @@ set -euo pipefail
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--rerun-args '--update -M400000' \
|
||||
--log-found 'More than 400000 bytes have been transferred.. giving up' \
|
||||
--log-not-found 'could not restore' \
|
||||
--found bigtrunc/slow.bin \
|
||||
--file-min-bytes bigtrunc/slow.bin 655360 \
|
||||
--file-min-bytes bigtrunc/fast.bin 655360 \
|
||||
|
||||
@@ -79,6 +79,29 @@ done
|
||||
# 65535 is a valid port the old "< 65535" bound refused
|
||||
web_accepted 65535
|
||||
|
||||
# --- htsserver returns from main() instead of blocking at exit -------------
|
||||
# The exit wait must exclude exactly the threads that never return (#753).
|
||||
# $tmp has no lang.def, so the server fails right after announcing and main()
|
||||
# reaches the wait on its own; rc, not EXITED, carries the verdict.
|
||||
|
||||
web_exits() {
|
||||
local rc=0
|
||||
|
||||
: >"$tmp/exit.log"
|
||||
run_with_timeout 30 htsserver "$tmp" "$@" >"$tmp/exit.log" 2>&1 || rc=$?
|
||||
grep -q "^EXITED" "$tmp/exit.log" ||
|
||||
! echo "FAIL: #753: htsserver ${*:-(no options)} never finished serving" || exit 1
|
||||
# exactly 1, not merely "not 124": a crash or an assertf abort also escapes
|
||||
# the wait, and would otherwise read as a pass
|
||||
test "$rc" -eq 1 ||
|
||||
! echo "FAIL: #753: htsserver ${*:-(no options)} exited $rc, want 1" || exit 1
|
||||
}
|
||||
|
||||
# no pinger: the excluded count went negative
|
||||
web_exits
|
||||
# a pinger, which never returns and so must stay excluded
|
||||
web_exits --ppid $$
|
||||
|
||||
# --- proxytrack <proxy-addr:port> <ICP-addr:port> --------------------------
|
||||
# A bad argument falls through to the usage screen; it had no range check at
|
||||
# all, so 65616 quietly listened on port 80. A valid one binds and blocks.
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A --warc crawl writes a standards-conformant WARC/1.1 archive, and an
|
||||
# --update re-crawl of an unchanged (all-304) site emits revisit records.
|
||||
# The stdlib validator (no warcio) is the real gate: it byte-compares the
|
||||
# fresh page.html response body against what the server served, asserts the
|
||||
# encoding headers were stripped and the payload digest matches, then checks
|
||||
# the update pass turned the unchanged assets into revisits (no full response).
|
||||
# A --warc crawl writes a standards-conformant WARC/1.1 archive, and a re-crawl
|
||||
# of an unchanged (all-304) site emits revisit records. The stdlib validator (no
|
||||
# warcio) is the real gate: it byte-compares the fresh page.html response body
|
||||
# against what the server served, asserts the encoding headers were stripped and
|
||||
# the payload digest matches, then checks the second pass turned the unchanged
|
||||
# assets into revisits (no full response).
|
||||
#
|
||||
# That second pass archives under a name of its own, which is also the recovery
|
||||
# the engine points at when it refuses to strand a previous archive (#759).
|
||||
|
||||
set -eu
|
||||
|
||||
@@ -14,8 +17,10 @@ set -eu
|
||||
# page.html body served by tests/local-server.py (route_mini304_page).
|
||||
export WARC_VALIDATE_BODY="page.html=3c68746d6c3e3c626f64793e74696e7920636163686561626c6520706167653c2f626f64793e3c2f68746d6c3e0a"
|
||||
export WARC_VALIDATE_NORESP="index.html page.html"
|
||||
export WARC_VALIDATE_UPDATE="warc-update.warc.gz"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun --warc-validate \
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --warc-validate \
|
||||
--rerun-args '--warc-file warc-update' \
|
||||
--log-found 'no files updated' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out
|
||||
|
||||
@@ -5,11 +5,17 @@
|
||||
# real gate: STORE-mode entries, the fixed layout, recomputed sha256 digests and
|
||||
# the datapackage-digest chain (plus py-wacz/pywb when importable). The crawl
|
||||
# skips cleanly on a build without OpenSSL (no conformant SHA-256 -> no package).
|
||||
#
|
||||
# The cacheless (-C0) second pass repackages over the .wacz the first one left:
|
||||
# the clobber, not the happy rename, is what breaks when a platform refuses to
|
||||
# rename onto an existing file (#726). A cache-served pass keeps the previous
|
||||
# package instead, which is 97_local-warc-update-keep.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --wacz-validate \
|
||||
--rerun-args '-C0' --log-not-found 'could not finalize' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --wacz
|
||||
|
||||
@@ -94,10 +94,7 @@ expect_counts_match() {
|
||||
echo "OK"
|
||||
}
|
||||
lines() { printf '%s\n' "$@" | sort; }
|
||||
# A connection killed before the status line surfaces differently per platform
|
||||
# (macOS truncates the file, #748; Linux leaves it), so reset.bin is asserted on
|
||||
# its own and kept out of the exact lists.
|
||||
listed_but_reset() { listed "$1" | grep -v '/reset\.bin$' | sort; }
|
||||
digest() { cksum <"$1" | tr -d '[:space:]'; }
|
||||
|
||||
# --- pass 1: nothing to compare against --------------------------------------
|
||||
httrack "${common[@]}" -O "$out" --purge-old=0 "${base}/changes/index.html" \
|
||||
@@ -120,6 +117,7 @@ grep -aq "first crawl" "${out}/hts-log.txt" || {
|
||||
}
|
||||
|
||||
host="127.0.0.1_${port}"
|
||||
resetdigest=$(digest "${out}/${host}/changes/reset.bin")
|
||||
# A leftover from some earlier failed attempt, at a name pass 2 fetches for the
|
||||
# first time: on disk, but never part of the previous mirror, so it is new.
|
||||
printf 'leftover junk' >"${out}/${host}/changes/fresh.html"
|
||||
@@ -144,21 +142,19 @@ expect "gone is doomed.html and the old redirect target" \
|
||||
expect "changed is exactly the four moved resources" \
|
||||
"$(lines "${host}/changes/coded.bin" "${host}/changes/index.html" \
|
||||
"${host}/changes/moved.bin" "${host}/changes/moved.html")" \
|
||||
"$(listed_but_reset changed)"
|
||||
"$(listed changed | sort)"
|
||||
# Re-served byte for byte, 200, no Last-Modified, no ETag. flaky.bin gets there
|
||||
# through a failed transfer and a retry, so its file is notified twice;
|
||||
# redirtarget.html arrives behind a 302. sized.html has a byte-identical
|
||||
# payload, but the rewritten link to the renamed redirect target makes the file
|
||||
# on disk change length.
|
||||
expect "unchanged is exactly the six stable resources" \
|
||||
# on disk change length. reset.bin never completes a transfer (#746), so its
|
||||
# previous copy stands.
|
||||
expect "unchanged is exactly the seven stable resources" \
|
||||
"$(lines "${host}/changes/codedstable.bin" "${host}/changes/flaky.bin" \
|
||||
"${host}/changes/redirtarget.html" "${host}/changes/sized.html" \
|
||||
"${host}/changes/stable.bin" "${host}/changes/stable.html")" \
|
||||
"$(listed_but_reset unchanged)"
|
||||
# reset.bin never completes a transfer, so it drops out of new.lst with its
|
||||
# mirrored copy still there. Whatever else it is, it is not a deletion.
|
||||
expect "a failed re-fetch is never reported gone" "" \
|
||||
"$(listed gone | grep '/reset\.bin$' || true)"
|
||||
"${host}/changes/redirtarget.html" "${host}/changes/reset.bin" \
|
||||
"${host}/changes/sized.html" "${host}/changes/stable.bin" \
|
||||
"${host}/changes/stable.html")" \
|
||||
"$(listed unchanged | sort)"
|
||||
expect_counts_match "pass 2 counts match its lists"
|
||||
|
||||
# Every mirrored file appears once and only once, across all four buckets.
|
||||
@@ -192,19 +188,15 @@ test -f "${out}/${host}/changes/doomed.html" || {
|
||||
exit 1
|
||||
}
|
||||
echo "OK"
|
||||
printf '[a failed transfer keeps its file] ..\t'
|
||||
test -f "${out}/${host}/changes/reset.bin" || {
|
||||
echo "FAIL: reset.bin lost its previous copy"
|
||||
exit 1
|
||||
}
|
||||
echo "OK"
|
||||
expect "a failed transfer keeps its bytes" "$resetdigest" \
|
||||
"$(digest "${out}/${host}/changes/reset.bin")"
|
||||
|
||||
# --- pass 3: the same run with purging on ------------------------------------
|
||||
httrack "${common[@]}" -O "$out" --update "${base}/changes/index.html" \
|
||||
>"${tmpdir}/log3" 2>&1
|
||||
expect "the report says files were purged" "true" "$(field purged)"
|
||||
expect "gone is the page that only pass 2 linked" \
|
||||
"${host}/changes/transient.html" "$(listed_but_reset gone)"
|
||||
"${host}/changes/transient.html" "$(listed gone | sort)"
|
||||
printf '[a purged file is off disk] ..\t'
|
||||
test ! -f "${out}/${host}/changes/transient.html" || {
|
||||
echo "FAIL: transient.html survived --purge-old"
|
||||
|
||||
25
tests/96_local-refetch-keep.test
Normal file
25
tests/96_local-refetch-keep.test
Normal file
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A re-fetch cut mid-header must leave the mirrored file alone (#748) and keep
|
||||
# it out of the update purge (#746). err.bin is the control, an HTTP 500 on the
|
||||
# same resource already surviving both; stay.bin answers normally with a new
|
||||
# body, so a fix that stopped overwriting anything fails.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--rerun-args '--update' \
|
||||
--log-found 'after 2 retries at link .*keep/page\.html' \
|
||||
--log-found 'after 2 retries at link .*keep/data\.bin' \
|
||||
--file-matches 'keep/page.html' 'KEEP-PAGE-V1' \
|
||||
--file-not-matches 'keep/page.html' 'HTTP/1\.0 200' \
|
||||
--file-matches 'keep/data.bin' 'KEEP-BIN-V1' \
|
||||
--file-not-matches 'keep/data.bin' 'HTTP/1\.0 200' \
|
||||
--file-min-bytes 'keep/data.bin' 2048 \
|
||||
--file-matches 'keep/err.bin' 'KEEP-ERR-V1' \
|
||||
--file-min-bytes 'keep/err.bin' 2048 \
|
||||
--file-matches 'keep/stay.bin' 'KEEP-STAY-V2' \
|
||||
--file-min-bytes 'keep/stay.bin' 2048 \
|
||||
httrack 'BASEURL/keep/index.html' --retries=2
|
||||
15
tests/97_local-warc-update-keep.test
Normal file
15
tests/97_local-warc-update-keep.test
Normal file
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A cache-served second pass fetches no bodies, so it must leave the WARC, CDXJ
|
||||
# and WACZ of the first pass untouched (#759). The kept package is re-validated
|
||||
# to prove it still replays, not just that a same-named file is still there.
|
||||
# The one Error: line is the keep notice; a second means the pass also failed.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 1 --archive-kept-on-rerun \
|
||||
--rerun --wacz-validate --log-found 'WARC: this pass revisited' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --wacz
|
||||
24
tests/98_local-warc-segments.test
Normal file
24
tests/98_local-warc-segments.test
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The same keep/replace rule under --warc-max-size, where the archive is several
|
||||
# segments: a cache-served pass must leave every one alone, a cacheless pass must
|
||||
# swap every one in. Half a swap is the corruption one rename loop can produce,
|
||||
# and only a rotated archive exposes it (#759). --archive-min-files pins the two
|
||||
# segments plus the .cdx, so a shrinking crawl cannot quietly stop rotating.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# The one Error: line is the keep notice; a second means the pass also failed.
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 1 --archive-kept-on-rerun \
|
||||
--archive-min-files 3 --rerun --log-found 'WARC: this pass revisited' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --warc-cdx \
|
||||
--warc-max-size 600
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --archive-replaced-on-rerun \
|
||||
--archive-min-files 3 --rerun-args '-C0' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --warc-cdx \
|
||||
--warc-max-size 600
|
||||
17
tests/99_local-robots-error.test
Normal file
17
tests/99_local-robots-error.test
Normal file
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
# Issue #769: a failed robots.txt probe under -o1, with .txt assumed image/gif,
|
||||
# still drives the --sitemap fallback and substitutes no placeholder body.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# -Z: both the removed block's own log line and the failed probe that gates
|
||||
# this test's coverage are debug-level.
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'sitemapdir/orphan1.html' \
|
||||
--log-found 'No robots\.txt rules at' \
|
||||
--log-found 'listed by 127\.0\.0\.1:[0-9]+/sitemap\.xml' \
|
||||
--log-not-found 'Creating GIF dummy file' \
|
||||
httrack 'BASEURL/sitemapdir/start.html' --sitemap -r2 -o1 -Z \
|
||||
--assume 'txt=image/gif' -F 'errorrobots-agent'
|
||||
@@ -1,11 +1,12 @@
|
||||
# Committed binary fixture read by 01_zlib-cache-golden.test. List it
|
||||
# explicitly: automake does not expand wildcards in EXTRA_DIST, so a glob would
|
||||
# silently drop it from the dist tarball and break "make distcheck".
|
||||
EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
EXTRA_DIST = $(TESTS) renamefail.c crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
proxy-https-server.py socks5-server.py proxy-connect-server.py \
|
||||
proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \
|
||||
pty-resize.py \
|
||||
local-crawl.sh local-server.py testlib.sh server.crt server.key \
|
||||
local-crawl.sh local-server.py ftp-server.py testlib.sh \
|
||||
server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
server-root/fraglink/index.html server-root/fraglink/target.html \
|
||||
@@ -21,6 +22,15 @@ TESTS_ENVIRONMENT += BROTLI_ENABLED=$(BROTLI_ENABLED)
|
||||
TESTS_ENVIRONMENT += ZSTD_ENABLED=$(ZSTD_ENABLED)
|
||||
TESTS_ENVIRONMENT += V6_SUPPORT=$(V6_SUPPORT)
|
||||
TESTS_ENVIRONMENT += top_srcdir=$(top_srcdir)
|
||||
TESTS_ENVIRONMENT += RENAMEFAIL_LA=$(abs_builddir)/librenamefail.la
|
||||
TESTS_ENVIRONMENT += RENAMEFAIL_LIB=$(abs_builddir)/$(LT_CV_OBJDIR)/librenamefail.so
|
||||
|
||||
# rename() interposer for 01_engine-renameover.test; -rpath is what makes
|
||||
# libtool build a shared module rather than a static-only convenience library.
|
||||
check_LTLIBRARIES = librenamefail.la
|
||||
librenamefail_la_SOURCES = renamefail.c
|
||||
librenamefail_la_LDFLAGS = -module -avoid-version -rpath $(abs_builddir)
|
||||
librenamefail_la_LIBADD = $(DL_LIBS)
|
||||
|
||||
TEST_EXTENSIONS = .test
|
||||
# Run each .test through bash instead of execve()ing it. This lets "make check"
|
||||
@@ -30,6 +40,7 @@ TEST_EXTENSIONS = .test
|
||||
TEST_LOG_COMPILER = $(BASH)
|
||||
TESTS = \
|
||||
00_runnable.test \
|
||||
01_engine-addlink.test \
|
||||
01_engine-changes.test \
|
||||
01_engine-charset.test \
|
||||
01_engine-cmdline.test \
|
||||
@@ -48,6 +59,7 @@ TESTS = \
|
||||
01_engine-filterdual.test \
|
||||
01_engine-ftp-line.test \
|
||||
01_engine-ftp-userpass.test \
|
||||
01_engine-backswap.test \
|
||||
01_engine-cacheindex.test \
|
||||
01_engine-hashtable.test \
|
||||
01_engine-header.test \
|
||||
@@ -58,6 +70,7 @@ TESTS = \
|
||||
01_engine-identabs.test \
|
||||
01_engine-escape-room.test \
|
||||
01_engine-inplace-escape.test \
|
||||
01_engine-lastchar.test \
|
||||
01_engine-makeindex.test \
|
||||
01_engine-mime.test \
|
||||
01_engine-parse.test \
|
||||
@@ -73,6 +86,7 @@ TESTS = \
|
||||
01_engine-direnum.test \
|
||||
01_engine-cookieimport.test \
|
||||
01_engine-relative.test \
|
||||
01_engine-renameover.test \
|
||||
01_engine-robots.test \
|
||||
01_engine-savename.test \
|
||||
01_engine-selftest-dispatch.test \
|
||||
@@ -81,7 +95,9 @@ TESTS = \
|
||||
01_engine-status.test \
|
||||
01_engine-stripquery.test \
|
||||
01_engine-strsafe.test \
|
||||
01_engine-structcheck.test \
|
||||
01_engine-syscharset.test \
|
||||
01_engine-threadwait.test \
|
||||
01_engine-topindex.test \
|
||||
01_engine-urlhack.test \
|
||||
01_engine-unescape-bounds.test \
|
||||
@@ -194,6 +210,13 @@ TESTS = \
|
||||
92_local-proxytrack-ndx-fields.test \
|
||||
93_local-changes.test \
|
||||
94_local-single-file.test \
|
||||
95_local-sitemap.test
|
||||
95_local-sitemap.test \
|
||||
96_local-refetch-keep.test \
|
||||
97_local-warc-update-keep.test \
|
||||
98_local-warc-segments.test \
|
||||
99_local-robots-error.test \
|
||||
100_local-purge-longpath.test \
|
||||
101_local-update-stale-bak.test \
|
||||
102_local-ftp-refetch.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
227
tests/ftp-server.py
Normal file
227
tests/ftp-server.py
Normal file
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Minimal FTP server for the crawl tests: PASV only, binary RETR, LIST.
|
||||
|
||||
Prints "PORT <n>" once bound, like local-server.py. --mode-file is re-read
|
||||
before every transfer, so a test can flip one path's behaviour between passes
|
||||
without restarting the server and moving the port (which names the mirror
|
||||
directory). Each line is "<path> <mode>...", path "*" matching everything:
|
||||
|
||||
truncate send a prefix of the body, then drop the data connection
|
||||
empty open the data connection and send nothing
|
||||
norest answer REST with 500, so the client re-fetches from scratch
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
|
||||
|
||||
def reply(conn, text):
|
||||
conn.sendall((text + "\r\n").encode("utf-8", "replace"))
|
||||
|
||||
|
||||
class Session(threading.Thread):
|
||||
def __init__(self, conn, root, mode_file, log):
|
||||
threading.Thread.__init__(self, daemon=True)
|
||||
self.conn = conn
|
||||
self.root = root
|
||||
self.mode_file = mode_file
|
||||
self.log = log
|
||||
self.pasv = None
|
||||
self.rest = 0
|
||||
self.path = "/" # named by SIZE/RETR; REST carries no path
|
||||
|
||||
def modes(self):
|
||||
if not self.mode_file:
|
||||
return set()
|
||||
found = set()
|
||||
try:
|
||||
with open(self.mode_file, encoding="utf-8") as fp:
|
||||
for line in fp:
|
||||
words = line.split()
|
||||
if words and words[0] in ("*", self.path):
|
||||
found |= set(words[1:])
|
||||
except OSError:
|
||||
pass
|
||||
return found
|
||||
|
||||
# Resolve an FTP path argument under the root, refusing escapes.
|
||||
def resolve(self, arg):
|
||||
arg = arg.strip().strip('"')
|
||||
self.path = "/" + arg.lstrip("/")
|
||||
path = os.path.normpath(os.path.join(self.root, arg.lstrip("/")))
|
||||
if path != self.root and not path.startswith(self.root + os.sep):
|
||||
return None
|
||||
return path
|
||||
|
||||
def open_pasv(self):
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.bind(("127.0.0.1", 0))
|
||||
srv.listen(1)
|
||||
self.pasv = srv
|
||||
port = srv.getsockname()[1]
|
||||
return "227 Entering Passive Mode (127,0,0,1,%d,%d)" % (port >> 8, port & 0xFF)
|
||||
|
||||
def accept_data(self):
|
||||
if self.pasv is None:
|
||||
return None
|
||||
self.pasv.settimeout(30)
|
||||
try:
|
||||
data, _ = self.pasv.accept()
|
||||
except (OSError, socket.timeout):
|
||||
data = None
|
||||
self.pasv.close()
|
||||
self.pasv = None
|
||||
return data
|
||||
|
||||
def send_body(self, body, modes):
|
||||
data = self.accept_data()
|
||||
if data is None:
|
||||
reply(self.conn, "425 no data connection")
|
||||
return
|
||||
try:
|
||||
if "empty" in modes:
|
||||
pass
|
||||
elif "truncate" in modes:
|
||||
data.sendall(body[: max(1, len(body) // 8)])
|
||||
else:
|
||||
data.sendall(body)
|
||||
except OSError:
|
||||
pass
|
||||
data.close()
|
||||
if modes & {"empty", "truncate"}:
|
||||
reply(self.conn, "426 transfer aborted")
|
||||
else:
|
||||
reply(self.conn, "226 Transfer complete")
|
||||
|
||||
def do_retr(self, arg):
|
||||
path = self.resolve(arg)
|
||||
if path is None or not os.path.isfile(path):
|
||||
reply(self.conn, "550 no such file")
|
||||
return
|
||||
with open(path, "rb") as fp:
|
||||
body = fp.read()
|
||||
modes = self.modes()
|
||||
reply(self.conn, "150 opening binary mode")
|
||||
self.send_body(body[self.rest :], modes)
|
||||
self.rest = 0
|
||||
|
||||
def do_list(self, arg):
|
||||
if arg.startswith("-A"):
|
||||
arg = arg[2:].strip()
|
||||
path = self.resolve(arg or "/")
|
||||
if path is None or not os.path.isdir(path):
|
||||
reply(self.conn, "550 no such directory")
|
||||
return
|
||||
lines = []
|
||||
for name in sorted(os.listdir(path)):
|
||||
full = os.path.join(path, name)
|
||||
if os.path.isdir(full):
|
||||
lines.append("drwxr-xr-x 2 ftp ftp 4096 Jan 01 00:00 %s" % name)
|
||||
else:
|
||||
lines.append(
|
||||
"-rw-r--r-- 1 ftp ftp %d Jan 01 00:00 %s"
|
||||
% (os.path.getsize(full), name)
|
||||
)
|
||||
reply(self.conn, "150 opening ASCII mode")
|
||||
self.send_body(("\r\n".join(lines) + "\r\n").encode("utf-8"), set())
|
||||
|
||||
def dispatch(self, verb, arg):
|
||||
conn = self.conn
|
||||
if verb in ("USER", "PASS", "TYPE", "NOOP"):
|
||||
reply(conn, "200 ok")
|
||||
elif verb == "SYST":
|
||||
reply(conn, "215 UNIX Type: L8")
|
||||
elif verb == "PWD":
|
||||
reply(conn, '257 "/"')
|
||||
elif verb == "CWD":
|
||||
reply(conn, "250 ok")
|
||||
elif verb == "PASV":
|
||||
reply(conn, self.open_pasv())
|
||||
elif verb == "EPSV":
|
||||
reply(conn, "500 not supported")
|
||||
elif verb == "SIZE":
|
||||
path = self.resolve(arg)
|
||||
if path and os.path.isfile(path):
|
||||
reply(conn, "213 %d" % os.path.getsize(path))
|
||||
else:
|
||||
reply(conn, "550 no such file")
|
||||
elif verb == "REST":
|
||||
if "norest" in self.modes():
|
||||
reply(conn, "500 REST not understood")
|
||||
else:
|
||||
try:
|
||||
self.rest = int(arg)
|
||||
except ValueError:
|
||||
self.rest = 0
|
||||
reply(conn, "350 restarting")
|
||||
elif verb == "RETR":
|
||||
self.do_retr(arg)
|
||||
elif verb in ("LIST", "NLST"):
|
||||
self.do_list(arg)
|
||||
elif verb == "QUIT":
|
||||
reply(conn, "221 bye")
|
||||
return False
|
||||
else:
|
||||
reply(conn, "500 unknown command")
|
||||
return True
|
||||
|
||||
def run(self):
|
||||
conn = self.conn
|
||||
buf = b""
|
||||
try:
|
||||
reply(conn, "220 httrack test ftp")
|
||||
while True:
|
||||
while b"\r\n" not in buf:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
return
|
||||
buf += chunk
|
||||
line, buf = buf.split(b"\r\n", 1)
|
||||
line = line.decode("utf-8", "replace")
|
||||
self.log(line)
|
||||
verb, _, arg = line.partition(" ")
|
||||
if not self.dispatch(verb.upper(), arg):
|
||||
return
|
||||
except OSError:
|
||||
return
|
||||
finally:
|
||||
if self.pasv is not None:
|
||||
self.pasv.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--root", required=True)
|
||||
ap.add_argument("--mode-file")
|
||||
ap.add_argument("--log")
|
||||
args = ap.parse_args()
|
||||
|
||||
logfp = open(args.log, "a", encoding="utf-8") if args.log else None
|
||||
log_lock = threading.Lock()
|
||||
|
||||
def log(line):
|
||||
if logfp:
|
||||
with log_lock:
|
||||
logfp.write(line + "\n")
|
||||
logfp.flush()
|
||||
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
srv.bind(("127.0.0.1", 0))
|
||||
srv.listen(16)
|
||||
sys.stdout.reconfigure(newline="\n") # the launcher parses PORT, CRLF breaks it
|
||||
sys.stdout.write("PORT %d\n" % srv.getsockname()[1])
|
||||
sys.stdout.flush()
|
||||
|
||||
root = os.path.abspath(args.root)
|
||||
while True:
|
||||
conn, _ = srv.accept()
|
||||
Session(conn, root, args.mode_file, log).start()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -34,6 +34,15 @@
|
||||
# httrack via --cookies-file, to exercise preloaded cookies.
|
||||
# --rerun-dead re-runs with the server stopped: the no-data rollback must
|
||||
# restore the previous hts-cache generation byte-identical.
|
||||
# --archive-kept-on-rerun: the second pass must leave the first pass's
|
||||
# .warc[.gz]/.cdx/.wacz byte-identical, having no bodies to replace them (#759).
|
||||
# --archive-replaced-on-rerun is its mirror: every one of them must have been
|
||||
# rewritten. Both also require no *.tmp left behind, and take an optional
|
||||
# --archive-min-files N guarding against a scenario that silently stopped
|
||||
# producing the segments it means to check.
|
||||
# --plant-file/--plant-dir drop a regular file (holding $plant_poison) or a
|
||||
# directory at PATH under the host root between the passes, to hand the second
|
||||
# pass leftovers a killed run would have left (#758).
|
||||
|
||||
set -u
|
||||
|
||||
@@ -54,9 +63,15 @@ outdir_intl=
|
||||
rerun=
|
||||
rerun_args=
|
||||
rerun_dead=
|
||||
archive_kept=
|
||||
archive_replaced=
|
||||
archive_min_files=0
|
||||
tmpdir=
|
||||
serverpid=
|
||||
crawlpid=
|
||||
wacz_poisoned=
|
||||
wacz_poison="stale-wacz-that-a-second-pass-must-replace"
|
||||
plant_poison="stale-leftover-that-a-second-pass-must-clobber"
|
||||
|
||||
function warning {
|
||||
echo "** $*" >&2
|
||||
@@ -85,6 +100,18 @@ function cleanup {
|
||||
fi
|
||||
}
|
||||
|
||||
hostroot=
|
||||
function find_hostroot {
|
||||
local cand
|
||||
for cand in "${mirrorroot}/127.0.0.1_${port}" "${mirrorroot}/127.0.0.1"; do
|
||||
if test -d "$cand"; then
|
||||
hostroot="$cand"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
die "could not find host root under $out"
|
||||
}
|
||||
|
||||
function assert_equals {
|
||||
info "$1"
|
||||
if test ! "$2" == "$3"; then
|
||||
@@ -107,6 +134,7 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
|
||||
# --- parse leading control flags --------------------------------------------
|
||||
declare -a audit=()
|
||||
declare -a cookies=()
|
||||
declare -a plants=()
|
||||
scheme=http
|
||||
pos=0
|
||||
args=("$@")
|
||||
@@ -116,6 +144,13 @@ while test "$pos" -lt "$nargs"; do
|
||||
--debug) verbose=1 ;;
|
||||
--rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
|
||||
--rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
|
||||
# the second pass must leave the first pass's archive files untouched
|
||||
--archive-kept-on-rerun) archive_kept=1 ;;
|
||||
--archive-replaced-on-rerun) archive_replaced=1 ;; # ...or rewrite all of them
|
||||
--archive-min-files)
|
||||
pos=$((pos + 1))
|
||||
archive_min_files="${args[$pos]}"
|
||||
;;
|
||||
# validate the produced .warc.gz (see the validation block near the end)
|
||||
--warc-validate) warc_validate=1 ;;
|
||||
# validate the produced .wacz package (stdlib, plus py-wacz/pywb if present)
|
||||
@@ -139,6 +174,10 @@ while test "$pos" -lt "$nargs"; do
|
||||
pos=$((pos + 1))
|
||||
cookies+=("${args[$pos]}")
|
||||
;;
|
||||
--plant-file | --plant-dir)
|
||||
plants+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
;;
|
||||
--rerun-args)
|
||||
pos=$((pos + 1))
|
||||
rerun_args="${args[$pos]}"
|
||||
@@ -272,6 +311,47 @@ if test -n "$warc_validate"; then
|
||||
test -z "$w1" || cp "$w1" "${tmpdir}/warc-pass1.gz"
|
||||
fi
|
||||
|
||||
# Snapshot the archive files the second pass must keep (or must replace).
|
||||
declare -a kept_files=()
|
||||
if test -n "${archive_kept}${archive_replaced}"; then
|
||||
while read -r f; do
|
||||
test -n "$f" || continue
|
||||
cp "$f" "${tmpdir}/kept-${#kept_files[@]}" || die "could not snapshot $f"
|
||||
kept_files+=("$f")
|
||||
done < <(find "$mirrorroot" -maxdepth 2 \
|
||||
\( -name '*.warc.gz' -o -name '*.warc' -o -name '*.cdx' -o -name '*.wacz' \) \
|
||||
2>/dev/null | sort)
|
||||
test "${#kept_files[@]}" -gt 0 ||
|
||||
die "the first pass produced no archive to compare against"
|
||||
test "${#kept_files[@]}" -ge "$archive_min_files" ||
|
||||
die "only ${#kept_files[@]} archive file(s), wanted $archive_min_files: ${kept_files[*]}"
|
||||
fi
|
||||
|
||||
# Poison the first-pass .wacz when a second pass follows: repackaging moves the
|
||||
# new archive over it, so the marker must be gone afterwards (#726). Poisoning
|
||||
# beats comparing the two packages, which can come out byte-identical.
|
||||
if test -z "$archive_kept" && test -n "$wacz_validate" && test -n "${rerun}${rerun_args}"; then
|
||||
wacz_poisoned=$(find "$mirrorroot" -maxdepth 2 -name '*.wacz' 2>/dev/null | sort | tail -n1)
|
||||
test -z "$wacz_poisoned" || echo "$wacz_poison" >"$wacz_poisoned"
|
||||
fi
|
||||
|
||||
# --- plant leftovers the second pass has to deal with ------------------------
|
||||
if test "${#plants[@]}" -gt 0; then
|
||||
find_hostroot
|
||||
i=0
|
||||
while test "$i" -lt "${#plants[@]}"; do
|
||||
path="${hostroot}/${plants[$((i + 1))]}"
|
||||
info "planting ${plants[$i]} ${plants[$((i + 1))]}"
|
||||
if test "${plants[$i]}" = "--plant-dir"; then
|
||||
mkdir -p "$path" || die "could not create $path"
|
||||
else
|
||||
echo "$plant_poison" >"$path" || die "could not write $path"
|
||||
fi
|
||||
result "OK"
|
||||
i=$((i + 2))
|
||||
done
|
||||
fi
|
||||
|
||||
# --- optional second pass: re-mirror into the same dir (cache/update path) ----
|
||||
if test -n "$rerun"; then
|
||||
info "re-running httrack (update pass)"
|
||||
@@ -316,6 +396,29 @@ if test -n "$rerun_args"; then
|
||||
result "OK (second pass)"
|
||||
fi
|
||||
|
||||
# --- optional: did the second pass keep, or replace, the whole archive? ------
|
||||
if test "${#kept_files[@]}" -gt 0; then
|
||||
i=0
|
||||
for f in "${kept_files[@]}"; do
|
||||
if test -n "$archive_kept"; then
|
||||
info "checking the second pass kept $(basename "$f")"
|
||||
cmp -s "${tmpdir}/kept-${i}" "$f" ||
|
||||
die "$(basename "$f") was rewritten: the previous archive was destroyed"
|
||||
else
|
||||
info "checking the second pass replaced $(basename "$f")"
|
||||
cmp -s "${tmpdir}/kept-${i}" "$f" &&
|
||||
die "$(basename "$f") still holds the first pass's bytes"
|
||||
fi
|
||||
result "OK"
|
||||
i=$((i + 1))
|
||||
done
|
||||
# A leftover in-progress file is as bad: the next pass would silently eat it.
|
||||
info "checking no in-progress archive was left behind"
|
||||
leftover=$(find "$mirrorroot" -maxdepth 2 \( -name '*.warc.gz.tmp' -o -name '*.warc.tmp' \) 2>/dev/null | head -n1)
|
||||
test -z "$leftover" || die "left behind $leftover"
|
||||
result "OK"
|
||||
fi
|
||||
|
||||
# --- optional dead pass: server stopped, the cache must survive the rollback --
|
||||
if test -n "$rerun_dead"; then
|
||||
zip="${out}/hts-cache/new.zip"
|
||||
@@ -353,14 +456,7 @@ if test -n "$rerun_dead"; then
|
||||
fi
|
||||
|
||||
# --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
|
||||
hostroot=
|
||||
for cand in "${mirrorroot}/127.0.0.1_${port}" "${mirrorroot}/127.0.0.1"; do
|
||||
if test -d "$cand"; then
|
||||
hostroot="$cand"
|
||||
break
|
||||
fi
|
||||
done
|
||||
test -n "$hostroot" || die "could not find host root under $out"
|
||||
find_hostroot
|
||||
debug "host root: $hostroot"
|
||||
|
||||
# --- optional WARC validation (stdlib validator, no warcio) ------------------
|
||||
@@ -387,14 +483,18 @@ if test -n "$warc_validate"; then
|
||||
die "fresh WARC validation failed"
|
||||
result "OK"
|
||||
|
||||
# Final file: after an update pass the unchanged assets must be revisits.
|
||||
if test -n "$rerun"; then
|
||||
# After an update pass the unchanged assets must be revisits, in an archive
|
||||
# of the pass's own (WARC_VALIDATE_UPDATE): a revisit-only pass never
|
||||
# replaces the archive holding the bodies it would strand (#759).
|
||||
if test -n "${WARC_VALIDATE_UPDATE:-}"; then
|
||||
upd=$(find "$mirrorroot" -maxdepth 2 -name "$WARC_VALIDATE_UPDATE" 2>/dev/null | head -n1)
|
||||
test -n "$upd" || die "no $WARC_VALIDATE_UPDATE produced under $mirrorroot"
|
||||
declare -a revargs=(--expect-revisit)
|
||||
for sub in ${WARC_VALIDATE_NORESP:-}; do
|
||||
revargs+=(--no-response-for "$sub")
|
||||
done
|
||||
info "validating update WARC (revisits)"
|
||||
"$python" "$validator" "$(nativepath "$warc")" "${revargs[@]}" >&2 ||
|
||||
"$python" "$validator" "$(nativepath "$upd")" "${revargs[@]}" >&2 ||
|
||||
die "update WARC validation failed"
|
||||
result "OK"
|
||||
fi
|
||||
@@ -416,6 +516,12 @@ if test -n "$wacz_validate"; then
|
||||
fi
|
||||
die "no .wacz file produced under $mirrorroot"
|
||||
fi
|
||||
if test -n "$wacz_poisoned"; then
|
||||
info "checking the second pass replaced the .wacz"
|
||||
grep -q "$wacz_poison" "$wacz_poisoned" 2>/dev/null &&
|
||||
die "stale .wacz kept: $wacz_poisoned"
|
||||
result "OK"
|
||||
fi
|
||||
validator=$(nativepath "${testdir}/wacz-validate.py")
|
||||
info "validating WACZ package"
|
||||
"$python" "$validator" "$(nativepath "$wacz")" >&2 || die "WACZ validation failed"
|
||||
|
||||
@@ -555,10 +555,22 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
# ... and one that points the sitemap at the site root, to check a subtree
|
||||
# crawl is not widened by where the site chooses to put its sitemap.
|
||||
SCOPE_SITEMAP_UA = "scopesitemap"
|
||||
# ... and one whose probe fails, with a body past the 1070 bytes that
|
||||
# issue #769's over-read needed.
|
||||
ERROR_ROBOTS_UA = "errorrobots"
|
||||
|
||||
def route_robots(self):
|
||||
# The Sitemap: record is group-independent; only --sitemap acts on it.
|
||||
ua = self.headers.get("User-Agent") or ""
|
||||
if self.ERROR_ROBOTS_UA in ua:
|
||||
body = b"# " + b"x" * 4000 + b"\n"
|
||||
self.send_response(404, "Not Found")
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
return
|
||||
host = self.headers.get("Host")
|
||||
body = "User-agent: *\nDisallow:\n"
|
||||
if self.DENY_DECLARED_UA in ua:
|
||||
@@ -1000,6 +1012,62 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
v = 1 if self.refetch_pass() == 1 else 2
|
||||
self.send_raw(b"<html><body><p>STAY-V%d</p></body></html>" % v, "text/html")
|
||||
|
||||
# --- re-fetch cut mid-header, so nothing is stored (#746, #748) ---------
|
||||
KEEP_PAGE = b"<html><body><p>KEEP-PAGE-V1</p></body></html>"
|
||||
KEEP_BIN = b"KEEP-BIN-V1\n" + b"\x51\x52\x53\x54" * 512
|
||||
KEEP_ERR = b"KEEP-ERR-V1\n" + b"\x61\x62\x63\x64" * 512
|
||||
|
||||
def send_cut_headers(self):
|
||||
"""Hang up mid-header: the response never becomes parseable."""
|
||||
self.close_connection = True
|
||||
try:
|
||||
self.wfile.write(b"HTTP/1.0 200 OK\r\nContent-Ty")
|
||||
self.wfile.flush()
|
||||
except OSError:
|
||||
pass
|
||||
self.connection.close()
|
||||
|
||||
def route_keep_index(self):
|
||||
self.refetch_pass()
|
||||
self.send_html(
|
||||
'\t<a href="page.html">page</a>\n'
|
||||
'\t<a href="data.bin">data</a>\n'
|
||||
'\t<a href="err.bin">err</a>\n'
|
||||
'\t<a href="stay.bin">stay</a>\n'
|
||||
)
|
||||
|
||||
def route_keep_page(self):
|
||||
if self.refetch_pass() == 1:
|
||||
self.send_raw(self.KEEP_PAGE, "text/html")
|
||||
else:
|
||||
self.send_cut_headers()
|
||||
|
||||
def route_keep_data(self):
|
||||
if self.refetch_pass() == 1:
|
||||
self.send_raw(self.KEEP_BIN, "application/octet-stream")
|
||||
else:
|
||||
self.send_cut_headers()
|
||||
|
||||
# Control: an HTTP error on the same resource already keeps the copy, so
|
||||
# the cut-header routes above must end up indistinguishable from it.
|
||||
def route_keep_err(self):
|
||||
if self.refetch_pass() == 1:
|
||||
self.send_raw(self.KEEP_ERR, "application/octet-stream")
|
||||
else:
|
||||
self.send_response(500)
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
# Control: answers normally, with a new body on pass 2, so a fix that
|
||||
# stopped overwriting mirrored files altogether would be caught.
|
||||
def route_keep_stay(self):
|
||||
v = 1 if self.refetch_pass() == 1 else 2
|
||||
self.send_raw(
|
||||
b"KEEP-STAY-V%d\n" % v + b"\x71\x72\x73\x74" * 512,
|
||||
"application/octet-stream",
|
||||
)
|
||||
|
||||
# Echo what httrack advertised, so a crawl can assert the header.
|
||||
def route_codec_ae(self):
|
||||
self.send_raw(
|
||||
@@ -1950,6 +2018,11 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/uptrunc/page.html": route_uptrunc_page,
|
||||
"/uptrunc/file.bin": route_uptrunc_file,
|
||||
"/uptrunc/stay.html": route_uptrunc_stay,
|
||||
"/keep/index.html": route_keep_index,
|
||||
"/keep/page.html": route_keep_page,
|
||||
"/keep/data.bin": route_keep_data,
|
||||
"/keep/err.bin": route_keep_err,
|
||||
"/keep/stay.bin": route_keep_stay,
|
||||
"/types/index.html": route_types_index,
|
||||
"/types/control.php": route_types,
|
||||
"/types/photo.png": route_types,
|
||||
|
||||
41
tests/renamefail.c
Normal file
41
tests/renamefail.c
Normal file
@@ -0,0 +1,41 @@
|
||||
/* Borrows Windows' rename() for 01_engine-renameover.test, since POSIX cannot
|
||||
reach hts_rename_over()'s unlink fallback: an existing target is refused with
|
||||
EEXIST, and RENAMEFAIL_MODE=locked reports EACCES instead, as the CRT does
|
||||
for a source another process holds. */
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#include <dlfcn.h>
|
||||
#include <errno.h>
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* The tree builds with -fvisibility=hidden, which would hide the interposer. */
|
||||
#define SHIM_EXPORT __attribute__((visibility("default")))
|
||||
|
||||
SHIM_EXPORT int rename(const char *oldpath, const char *newpath);
|
||||
|
||||
SHIM_EXPORT int rename(const char *oldpath, const char *newpath) {
|
||||
static int (*real_rename)(const char *, const char *) = NULL;
|
||||
const char *const mode = getenv("RENAMEFAIL_MODE");
|
||||
const int locked = mode != NULL && strcmp(mode, "locked") == 0;
|
||||
struct stat st;
|
||||
|
||||
if (locked) {
|
||||
errno = EACCES;
|
||||
return -1;
|
||||
}
|
||||
if (stat(newpath, &st) == 0) {
|
||||
errno = EEXIST;
|
||||
return -1;
|
||||
}
|
||||
if (real_rename == NULL) {
|
||||
*(void **) &real_rename = dlsym(RTLD_NEXT, "rename");
|
||||
if (real_rename == NULL) {
|
||||
errno = ENOSYS;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return real_rename(oldpath, newpath);
|
||||
}
|
||||
Reference in New Issue
Block a user