mirror of
https://github.com/xroche/httrack.git
synced 2026-07-25 10:09:56 +03:00
Compare commits
2 Commits
3.49.14
...
feat/webht
| Author | SHA1 | Date | |
|---|---|---|---|
|
4ae6124257 | ||
|
|
76dcf61d20 |
13
configure.ac
13
configure.ac
@@ -1,6 +1,6 @@
|
||||
AC_PREREQ([2.71])
|
||||
|
||||
AC_INIT([httrack], [3.49.14], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_INIT([httrack], [3.49.13], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_COPYRIGHT([
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2015 Xavier Roche and other contributors
|
||||
@@ -29,12 +29,13 @@ AC_CONFIG_SRCDIR(src/httrack.c)
|
||||
AC_CONFIG_MACRO_DIR([m4])
|
||||
AC_CONFIG_HEADERS(config.h)
|
||||
AM_INIT_AUTOMAKE([subdir-objects])
|
||||
# 3:6:0: 3.49.14 grows two installed structs from the inside (htsoptstate sits mid-
|
||||
# httrackp, htsblk mid-lien_back), so the fields after them shift: httrackp.cookies_file
|
||||
# +8, lien_back.http11 +48. Soname deliberately stays .so.3: the shifted fields are
|
||||
# 3.49-era additions no external consumer reaches, and a libhttrack3 rename isn't worth it.
|
||||
# 3:5:0: 3.49.13 leaves every installed struct layout and the exported symbol set
|
||||
# untouched vs 3.49.12; the 2 GB (_stat64) and UTF-8 export widenings are _WIN32-only,
|
||||
# and INTsys widened on POSIX (#600) but sits in no installed struct. Stays soname
|
||||
# .so.3; bump revision. x32 alone changes layout (LLint was 4 bytes there, #524), and
|
||||
# is not a release architecture.
|
||||
# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
|
||||
VERSION_INFO="3:6:0"
|
||||
VERSION_INFO="3:5:0"
|
||||
AM_MAINTAINER_MODE
|
||||
AC_USE_SYSTEM_EXTENSIONS
|
||||
|
||||
|
||||
8
debian/changelog
vendored
8
debian/changelog
vendored
@@ -1,11 +1,3 @@
|
||||
httrack (3.49.14-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: WARC/WACZ archive output, Windows long-path support,
|
||||
named -%F footer fields, and encoding fixes for non-ASCII project paths;
|
||||
full list in history.txt.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Fri, 24 Jul 2026 08:01:43 +0200
|
||||
|
||||
httrack (3.49.13-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: SOCKS5 and CONNECT proxy support, brotli and zstd
|
||||
|
||||
17
history.txt
17
history.txt
@@ -4,23 +4,6 @@ HTTrack Website Copier release history:
|
||||
|
||||
This file lists all changes and fixes that have been made for HTTrack
|
||||
|
||||
3.49-14
|
||||
+ New: WARC/1.1 archive output (--warc), with a sorted CDXJ index (--warc-cdx) and WACZ packaging (--wacz), also available from webhttrack (#668)
|
||||
+ New: -%F takes named footer fields such as {url}, {lastmodified}, {mime}, {charset} and {status} instead of a fixed layout (#667)
|
||||
+ New: a command-line guide organized by task ships with the offline documentation (#649)
|
||||
+ Fixed: on Windows, paths beyond MAX_PATH truncated files, and mirroring into a long or non-ASCII directory silently failed (#133)
|
||||
+ Fixed: the top index showed mojibake for non-ASCII project names and categories on Windows (#216)
|
||||
+ Fixed: webhttrack handed the engine the web form's charset rather than UTF-8, so a non-ASCII path mirrored into a mojibake directory (#629)
|
||||
+ Fixed: a non-ASCII single -O left the logs and the cache in a mangled twin directory on Windows (#630)
|
||||
+ Fixed: a path-ceiling truncation dropped the .delayed marker, losing the file (#623)
|
||||
+ Fixed: an oversized -%F footer aborted the crawl instead of being skipped (#669)
|
||||
+ Fixed: a rejected 206 resume could loop and lose the file rather than refetch it whole (#581)
|
||||
+ Fixed: default-port stripping was scheme-blind and dropped explicit ports from https and ftp URLs, and a :80 written with leading zeros mangled the host (#627, #638)
|
||||
+ Fixed: -K silently reset the -c socket count (#650)
|
||||
+ Fixed: signed-shift undefined behaviour in the zip-repair local-header read (#639)
|
||||
+ Changed: the offline documentation drops stale facts, gains an Android help page, and documents the filter wildcards and the real long option forms
|
||||
+ Changed: multiple internal hardening, test and CI improvements
|
||||
|
||||
3.49-13
|
||||
+ New: SOCKS5 proxy support, with scheme-aware -P URLs (socks5://, socks5h://, connect://) and plain HTTP tunneled through a CONNECT-only proxy (#563, #564)
|
||||
+ New: decode brotli and zstd content codings, advertised over TLS only as browsers do (#556)
|
||||
|
||||
@@ -473,15 +473,6 @@ then:</small><br>
|
||||
lifts the built-in caps; use it only against infrastructure you are allowed to
|
||||
load.</small></p>
|
||||
|
||||
<h4>Save a WARC archive of the crawl</h4>
|
||||
<p><tt>httrack https://example.com/ --warc --path mydir</tt><br>
|
||||
<small>Writes a standard WARC/1.1 file (<tt>httrack-<timestamp>.warc.gz</tt>) in
|
||||
the project folder alongside the browsable mirror, not instead of it. Set the name
|
||||
with <tt>--warc-file NAME</tt> and split a large crawl with <tt>--warc-max-size N</tt>;
|
||||
add <tt>--warc-cdx</tt> for a sorted CDXJ index, or <tt>--wacz</tt> to bundle the
|
||||
archive, index and pages into one WACZ for replay tools such as
|
||||
replayweb.page.</small></p>
|
||||
|
||||
<h4>HTTrack as a fetch tool</h4>
|
||||
<p><tt>httrack --get https://host/file.bin --path tmp</tt><br>
|
||||
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
|
||||
|
||||
@@ -90,34 +90,35 @@ offline browser : copy websites to a local directory</p>
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
<b>-%x, --disable-passwords</b> ] [ <b>-%q,
|
||||
--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
|
||||
<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
|
||||
] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
|
||||
] [ <b>-bN, --cookies[=N]</b> ] [ <b>-%K, --cookies-file</b>
|
||||
] [ <b>-%Y, --why</b> ] [ <b>-u, --check-type[=N]</b> ] [
|
||||
<b>-j, --parse-java[=N]</b> ] [ <b>-sN, --robots[=N]</b> ] [
|
||||
<b>-%h, --http-10</b> ] [ <b>-%k, --keep-alive</b> ] [
|
||||
<b>-%z, --disable-compression</b> ] [ <b>-%B, --tolerant</b>
|
||||
] [ <b>-%s, --updatehack</b> ] [ <b>-%u, --urlhack</b> ] [
|
||||
] [ <b>-%r, --warc</b> ] [ <b>-LN, --long-names[=N]</b> ] [
|
||||
<b>-KN, --keep-links[=N]</b> ] [ <b>-x,
|
||||
--replace-external</b> ] [ <b>-%x, --disable-passwords</b> ]
|
||||
[ <b>-%q, --include-query-string</b> ] [ <b>-%g,
|
||||
--strip-query</b> ] [ <b>-o, --generate-errors</b> ] [
|
||||
<b>-X, --purge-old[=N]</b> ] [ <b>-%p, --preserve</b> ] [
|
||||
<b>-%T, --utf8-conversion</b> ] [ <b>-bN, --cookies[=N]</b>
|
||||
] [ <b>-%K, --cookies-file</b> ] [ <b>-%Y, --why</b> ] [
|
||||
<b>-u, --check-type[=N]</b> ] [ <b>-j, --parse-java[=N]</b>
|
||||
] [ <b>-sN, --robots[=N]</b> ] [ <b>-%h, --http-10</b> ] [
|
||||
<b>-%k, --keep-alive</b> ] [ <b>-%z,
|
||||
--disable-compression</b> ] [ <b>-%B, --tolerant</b> ] [
|
||||
<b>-%s, --updatehack</b> ] [ <b>-%u, --urlhack</b> ] [
|
||||
<b>-%A, --assume</b> ] [ <b>-@iN, --protocol[=N]</b> ] [
|
||||
<b>-%w, --disable-module</b> ] [ <b>-F, --user-agent</b> ] [
|
||||
<b>-%R, --referer</b> ] [ <b>-%E, --from</b> ] [ <b>-%F,
|
||||
--footer</b> ] [ <b>-%l, --language</b> ] [ <b>-%a,
|
||||
--accept</b> ] [ <b>-%X, --headers</b> ] [ <b>-C,
|
||||
--cache[=N]</b> ] [ <b>-k, --store-all-in-cache</b> ] [
|
||||
<b>-%r, --warc</b> ] [ <b>-%n, --do-not-recatch</b> ] [
|
||||
<b>-%v, --display</b> ] [ <b>-Q, --do-not-log</b> ] [ <b>-q,
|
||||
--quiet</b> ] [ <b>-z, --extra-log</b> ] [ <b>-Z,
|
||||
--debug-log</b> ] [ <b>-v, --verbose</b> ] [ <b>-f,
|
||||
--file-log</b> ] [ <b>-f2, --single-log</b> ] [ <b>-I,
|
||||
--index</b> ] [ <b>-%i, --build-top-index</b> ] [ <b>-%I,
|
||||
--search-index</b> ] [ <b>-pN, --priority[=N]</b> ] [ <b>-S,
|
||||
--stay-on-same-dir</b> ] [ <b>-D, --can-go-down</b> ] [
|
||||
<b>-U, --can-go-up</b> ] [ <b>-B, --can-go-up-and-down</b> ]
|
||||
[ <b>-a, --stay-on-same-address</b> ] [ <b>-d,
|
||||
<b>-%n, --do-not-recatch</b> ] [ <b>-%v, --display</b> ] [
|
||||
<b>-Q, --do-not-log</b> ] [ <b>-q, --quiet</b> ] [ <b>-z,
|
||||
--extra-log</b> ] [ <b>-Z, --debug-log</b> ] [ <b>-v,
|
||||
--verbose</b> ] [ <b>-f, --file-log</b> ] [ <b>-f2,
|
||||
--single-log</b> ] [ <b>-I, --index</b> ] [ <b>-%i,
|
||||
--build-top-index</b> ] [ <b>-%I, --search-index</b> ] [
|
||||
<b>-pN, --priority[=N]</b> ] [ <b>-S, --stay-on-same-dir</b>
|
||||
] [ <b>-D, --can-go-down</b> ] [ <b>-U, --can-go-up</b> ] [
|
||||
<b>-B, --can-go-up-and-down</b> ] [ <b>-a,
|
||||
--stay-on-same-address</b> ] [ <b>-d,
|
||||
--stay-on-same-domain</b> ] [ <b>-l, --stay-on-same-tld</b>
|
||||
] [ <b>-e, --go-everywhere</b> ] [ <b>-%H,
|
||||
--debug-headers</b> ] [ <b>-%!,
|
||||
@@ -648,6 +649,18 @@ don’t wait) (--cached-delayed-type-check)</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%r</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
|
||||
sets the output name (--warc)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%t</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
@@ -1129,20 +1142,6 @@ update before) (--cache[=N])</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%r</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
|
||||
sets the output name, --warc-max-size N rotates segments
|
||||
past N bytes, --warc-cdx also writes a sorted CDXJ index,
|
||||
--wacz packages it all as a WACZ file (--warc)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%n</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "23 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "22 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -40,6 +40,7 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-%r, \-\-warc\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
[ \fB\-KN, \-\-keep\-links[=N]\fR ]
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
@@ -74,7 +75,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%X, \-\-headers\fR ]
|
||||
[ \fB\-C, \-\-cache[=N]\fR ]
|
||||
[ \fB\-k, \-\-store\-all\-in\-cache\fR ]
|
||||
[ \fB\-%r, \-\-warc\fR ]
|
||||
[ \fB\-%n, \-\-do\-not\-recatch\fR ]
|
||||
[ \fB\-%v, \-\-display\fR ]
|
||||
[ \fB\-Q, \-\-do\-not\-log\fR ]
|
||||
@@ -198,6 +198,8 @@ delayed type check, don't make any link test but wait for files download to star
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%r
|
||||
write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name (\-\-warc)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
@@ -277,8 +279,6 @@ additional HTTP header line (\-%X "X\-Magic: 42" (\-\-headers <param>)
|
||||
create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before) (\-\-cache[=N])
|
||||
.IP \-k
|
||||
store all files in cache (not useful if files on disk) (\-\-store\-all\-in\-cache)
|
||||
.IP \-%r
|
||||
write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name, \-\-warc\-max\-size N rotates segments past N bytes, \-\-warc\-cdx also writes a sorted CDXJ index, \-\-wacz packages it all as a WACZ file (\-\-warc)
|
||||
.IP \-%n
|
||||
do not re\-download locally erased files (\-\-do\-not\-recatch)
|
||||
.IP \-%v
|
||||
|
||||
@@ -116,13 +116,6 @@ const char *hts_optalias[][4] = {
|
||||
"load extra cookies from a Netscape cookies.txt"},
|
||||
{"warc", "-%r", "single", "write an ISO-28500 WARC/1.1 archive of the crawl"},
|
||||
{"warc-file", "-%rf", "param1", "write a WARC archive to the given base name"},
|
||||
{"warc-max-size", "-%rs", "param1",
|
||||
"rotate the WARC archive once a segment passes N bytes (0: single file)"},
|
||||
{"warc-cdx", "-%rc", "single",
|
||||
"write a sorted CDXJ index next to the WARC archive"},
|
||||
{"warc-cdxj", "-%rc", "single", ""},
|
||||
{"wacz", "-%rz", "single",
|
||||
"package the WARC archive, CDXJ index and pages as a WACZ file"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
|
||||
@@ -750,20 +750,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
fexist_utf8(back[p].url_sav))
|
||||
filenote(&opt->state.strc, back[p].url_sav, NULL);
|
||||
}
|
||||
/* Keep the compressed spool so the WARC record stores the body
|
||||
verbatim (Content-Encoding preserved) instead of unlinking it.
|
||||
*/
|
||||
if (StringNotEmpty(opt->warc_file)) {
|
||||
warc_adopt_rawspool(&back[p].r, back[p].tmpfile);
|
||||
if (back[p].r.warc_rawpath != NULL)
|
||||
back[p].tmpfile =
|
||||
NULL; /* adopted: freed via warc_free_request */
|
||||
}
|
||||
/* ensure that no remaining temporary file exists */
|
||||
if (back[p].tmpfile != NULL) {
|
||||
unlink(back[p].tmpfile);
|
||||
back[p].tmpfile = NULL;
|
||||
}
|
||||
unlink(back[p].tmpfile);
|
||||
back[p].tmpfile = NULL;
|
||||
}
|
||||
// stats
|
||||
HTS_STAT.total_packed += back[p].compressed_size;
|
||||
@@ -1073,9 +1062,6 @@ void back_copy_static(const lien_back * src, lien_back * dst) {
|
||||
dst->r.headers = NULL;
|
||||
dst->r.warc_reqhdr = NULL;
|
||||
dst->r.warc_resphdr = NULL;
|
||||
dst->r.warc_rawpath =
|
||||
NULL; /* the spool stays owned by src (no double-unlink) */
|
||||
dst->r.warc_truncated = 0;
|
||||
dst->r.out = NULL;
|
||||
dst->r.location = dst->location_buffer;
|
||||
dst->r.fp = NULL;
|
||||
@@ -1141,8 +1127,6 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
|
||||
(*dst)->r.out = NULL;
|
||||
(*dst)->r.warc_reqhdr = NULL;
|
||||
(*dst)->r.warc_resphdr = NULL;
|
||||
(*dst)->r.warc_rawpath = NULL;
|
||||
(*dst)->r.warc_truncated = 0;
|
||||
(*dst)->r.location = (*dst)->location_buffer;
|
||||
(*dst)->r.fp = NULL;
|
||||
(*dst)->r.soc = INVALID_SOCKET;
|
||||
@@ -2535,19 +2519,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
|
||||
for (i = 0; i < (unsigned int) back_max; i++) {
|
||||
if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
|
||||
/* A cap-truncated body is deliberate, not broken: archive what arrived
|
||||
with WARC-Truncated before the abort overwrites the slot's real 2xx
|
||||
status. HTTrack still treats the slot as incomplete afterwards. */
|
||||
if (StringNotEmpty(opt->warc_file) && back[i].r.statuscode > 0 &&
|
||||
back[i].r.warc_resphdr != NULL && back[i].r.size > 0 &&
|
||||
!(back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))) {
|
||||
if (back[i].r.is_write && back[i].r.out != NULL)
|
||||
fflush(back[i].r.out);
|
||||
back[i].r.warc_truncated = (limit == HTS_MIRROR_LIMIT_SIZE)
|
||||
? WARC_TRUNC_LENGTH
|
||||
: WARC_TRUNC_TIME;
|
||||
warc_write_backtransaction(opt, &back[i]);
|
||||
}
|
||||
if (back[i].r.soc != INVALID_SOCKET) {
|
||||
deletehttp(&back[i].r);
|
||||
}
|
||||
|
||||
@@ -39,9 +39,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsglobal.h"
|
||||
#include "htslib.h"
|
||||
#include "htscore.h"
|
||||
#ifdef _WIN32
|
||||
#include "htscharset.h" /* hts_pathToUCS2, hts_convertUCS2StringToUTF8 */
|
||||
#endif
|
||||
|
||||
/* END specific definitions */
|
||||
|
||||
@@ -203,36 +200,28 @@ char *cookie_nextfield(char *a) {
|
||||
return a;
|
||||
}
|
||||
|
||||
// Read cookies.txt (+ copied IE cookies *@*.txt on Windows); !=0 on error.
|
||||
// lire cookies.txt
|
||||
// lire également (Windows seulement) les *@*.txt (cookies IE copiés)
|
||||
// !=0 : erreur
|
||||
int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
char buffer[8192];
|
||||
|
||||
// Merge any IE cookies first
|
||||
// Fusionner d'abord les éventuels cookies IE
|
||||
#ifdef _WIN32
|
||||
{
|
||||
WIN32_FIND_DATAW find;
|
||||
WIN32_FIND_DATAA find;
|
||||
HANDLE h;
|
||||
char pth[HTS_URLMAXSIZE];
|
||||
LPWSTR wpth;
|
||||
char pth[MAX_PATH + 32];
|
||||
|
||||
strcpybuff(pth, fpath);
|
||||
strcatbuff(pth, "*@*.txt");
|
||||
// Wide glob so a long or non-ASCII IE-cookie folder is scanned (#133).
|
||||
wpth = hts_pathToUCS2(pth);
|
||||
h = wpth != NULL ? FindFirstFileW(wpth, &find) : INVALID_HANDLE_VALUE;
|
||||
freet(wpth);
|
||||
h = FindFirstFileA((char *) pth, &find);
|
||||
if (h != INVALID_HANDLE_VALUE) {
|
||||
do {
|
||||
if (!(find.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
|
||||
if (!(find.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM)) {
|
||||
// cFileName is UTF-16: convert to UTF-8 so the mirror path and
|
||||
// file wrappers stay UTF-8 (no CP_ACP mojibake).
|
||||
char *u = hts_convertUCS2StringToUTF8(find.cFileName, -1);
|
||||
FILE *fp =
|
||||
u != NULL
|
||||
? FOPEN(fconcat(catbuff, sizeof(catbuff), fpath, u), "rb")
|
||||
: NULL;
|
||||
FILE *fp = fopen(fconcat(catbuff, sizeof(catbuff), fpath, find.cFileName), "rb");
|
||||
|
||||
if (fp) {
|
||||
char cook_name[256];
|
||||
@@ -240,9 +229,11 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
|
||||
char domainpathpath[512];
|
||||
char dummy[512];
|
||||
|
||||
lien_adrfil af; // host + path (/)
|
||||
//
|
||||
lien_adrfil af; // chemin (/)
|
||||
int cookie_merged = 0;
|
||||
|
||||
//
|
||||
// Read all cookies
|
||||
while(!feof(fp)) {
|
||||
cook_name[0] = cook_value[0] = domainpathpath[0]
|
||||
@@ -271,11 +262,10 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
|
||||
}
|
||||
fclose(fp);
|
||||
if (cookie_merged)
|
||||
UNLINK(fconcat(catbuff, sizeof(catbuff), fpath, u));
|
||||
remove(fconcat(catbuff, sizeof(catbuff), fpath, find.cFileName));
|
||||
} // if fp
|
||||
freet(u);
|
||||
}
|
||||
} while (FindNextFileW(h, &find));
|
||||
} while(FindNextFileA(h, &find));
|
||||
FindClose(h);
|
||||
}
|
||||
}
|
||||
@@ -283,7 +273,7 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
|
||||
|
||||
// Ensuite, cookies.txt
|
||||
{
|
||||
FILE *fp = FOPEN(fconcat(catbuff, sizeof(catbuff), fpath, name), "rb");
|
||||
FILE *fp = fopen(fconcat(catbuff, sizeof(catbuff), fpath, name), "rb");
|
||||
|
||||
if (fp) {
|
||||
char BIGSTK line[8192];
|
||||
@@ -325,7 +315,7 @@ int cookie_save(t_cookie * cookie, const char *name) {
|
||||
if (strnotempty(cookie->data)) {
|
||||
char BIGSTK line[8192];
|
||||
#ifdef _WIN32
|
||||
FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), name), "wb");
|
||||
FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
|
||||
#else
|
||||
const int fd = open(fconv(catbuff, sizeof(catbuff), name),
|
||||
O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
|
||||
|
||||
@@ -171,13 +171,6 @@ extern LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int *pwsize);
|
||||
**/
|
||||
extern char *hts_convertUCS2StringToUTF8(LPWSTR woutput, int wsize);
|
||||
|
||||
/**
|
||||
* UTF-8 path to UCS-2 for the wide file/FindFirst APIs, \\?\-prefixed above
|
||||
* MAX_PATH (#133). Internal, not exported; caller frees.
|
||||
* This function is WIN32 specific.
|
||||
**/
|
||||
extern LPWSTR hts_pathToUCS2(const char *path);
|
||||
|
||||
/**
|
||||
* Convert current system codepage to UTF-8.
|
||||
* This function is WIN32 specific.
|
||||
|
||||
@@ -699,7 +699,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
int primary_len = 8192;
|
||||
|
||||
if (StringNotEmpty(opt->filelist)) {
|
||||
primary_len += max(0, fsize_utf8(StringBuff(opt->filelist)) * 2);
|
||||
primary_len += max(0, fsize(StringBuff(opt->filelist)) * 2);
|
||||
}
|
||||
primary_len += (int) strlen(url1) * 2;
|
||||
|
||||
@@ -856,19 +856,19 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
char *filelist_buff = NULL;
|
||||
size_t filelist_sz = 0;
|
||||
const char *filelist_err = NULL; /* failure reason, NULL on success */
|
||||
const LLint fs = fsize_utf8(StringBuff(opt->filelist));
|
||||
const LLint fs = fsize(StringBuff(opt->filelist));
|
||||
|
||||
if (fs < 0) {
|
||||
/* fsize() hides the cause; redo stat() for a precise errno (#49) */
|
||||
STRUCT_STAT st;
|
||||
filelist_err = STAT(StringBuff(opt->filelist), &st) != 0
|
||||
struct stat st;
|
||||
filelist_err = stat(StringBuff(opt->filelist), &st) != 0
|
||||
? strerror(errno)
|
||||
: "not a regular file";
|
||||
} else if ((filelist_sz = llint_to_size_t(fs)) == (size_t) -1) {
|
||||
filelist_err = "file too large";
|
||||
filelist_sz = 0;
|
||||
} else {
|
||||
FILE *fp = FOPEN(StringBuff(opt->filelist), "rb");
|
||||
FILE *fp = fopen(StringBuff(opt->filelist), "rb");
|
||||
|
||||
if (fp == NULL) {
|
||||
filelist_err = strerror(errno);
|
||||
@@ -976,9 +976,10 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
// statistiques
|
||||
if (opt->makestat) {
|
||||
makestat_fp = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-stats.txt"),
|
||||
"wb");
|
||||
makestat_fp =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-stats.txt"),
|
||||
"wb");
|
||||
if (makestat_fp != NULL) {
|
||||
fprintf(makestat_fp, "HTTrack statistics report, every minutes" LF LF);
|
||||
fflush(makestat_fp);
|
||||
@@ -986,9 +987,10 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
// tracking -- débuggage
|
||||
if (opt->maketrack) {
|
||||
maketrack_fp = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-track.txt"),
|
||||
"wb");
|
||||
maketrack_fp =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-track.txt"),
|
||||
"wb");
|
||||
if (maketrack_fp != NULL) {
|
||||
fprintf(maketrack_fp, "HTTrack tracking report, every minutes" LF LF);
|
||||
fflush(maketrack_fp);
|
||||
@@ -1637,8 +1639,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
|
||||
/* Remove file if being processed */
|
||||
if (is_loaded_from_file) {
|
||||
(void) UNLINK(
|
||||
fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), savename()));
|
||||
(void) unlink(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), savename()));
|
||||
is_loaded_from_file = 0;
|
||||
}
|
||||
|
||||
@@ -1951,8 +1952,8 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
} else */
|
||||
|
||||
/* External modules */
|
||||
if (opt->parsejava && (opt->parsejava & HTSPARSE_NO_CLASS) == 0 &&
|
||||
fexist_utf8(savename())) {
|
||||
if (opt->parsejava && (opt->parsejava & HTSPARSE_NO_CLASS) == 0
|
||||
&& fexist(savename())) {
|
||||
char BIGSTK buff_err_msg[1024];
|
||||
htsmoduleStruct BIGSTK str;
|
||||
|
||||
@@ -1993,7 +1994,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
} // text/html ou autre
|
||||
|
||||
/* Post-processing */
|
||||
if (fexist_utf8(savename())) {
|
||||
if (fexist(savename())) {
|
||||
usercommand(opt, 0, NULL, savename(), urladr(), urlfil());
|
||||
}
|
||||
|
||||
@@ -2084,16 +2085,18 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
//
|
||||
opt->state._hts_in_html_parsing = 3;
|
||||
//
|
||||
old_lst = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.lst"),
|
||||
"rb");
|
||||
old_lst =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"), "rb");
|
||||
if (old_lst) {
|
||||
const size_t sz = llint_to_size_t(fsize_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst")));
|
||||
new_lst = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst"),
|
||||
"rb");
|
||||
const size_t sz = llint_to_size_t(
|
||||
fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst")));
|
||||
new_lst =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"), "rb");
|
||||
if (new_lst != NULL && sz != (size_t) -1) {
|
||||
/* +1 for the NUL below: new.lst is read raw, and the strstr()
|
||||
that follows needs a terminated C string. */
|
||||
@@ -2113,9 +2116,9 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
strcpybuff(file, StringBuff(opt->path_html));
|
||||
strcatbuff(file, line + 1);
|
||||
file[strlen(file) - 1] = '\0';
|
||||
if (fexist_utf8(file)) { // toujours sur disque: virer
|
||||
if (fexist(file)) { // toujours sur disque: virer
|
||||
hts_log_print(opt, LOG_INFO, "Purging %s", file);
|
||||
UNLINK(file);
|
||||
remove(file);
|
||||
purge = 1;
|
||||
}
|
||||
}
|
||||
@@ -2136,8 +2139,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
|
||||
strcpybuff(file, StringBuff(opt->path_html));
|
||||
strcatbuff(file, line + 1);
|
||||
while ((strnotempty(file)) &&
|
||||
(RMDIR(file) == 0)) { // ok, éliminé (existait)
|
||||
while((strnotempty(file)) && (rmdir(file) == 0)) { // ok, éliminé (existait)
|
||||
purge = 1;
|
||||
if (opt->log) {
|
||||
hts_log_print(opt, LOG_INFO, "Purging directory %s/",
|
||||
@@ -2960,8 +2962,8 @@ static void postprocess_file(httrackp *opt, const char *save, const char *adr,
|
||||
if (adr != NULL && strcmp(adr, "primary") == 0) {
|
||||
adr = NULL;
|
||||
}
|
||||
if (save != NULL && opt != NULL && adr != NULL && adr[0] &&
|
||||
strnotempty(save) && fexist_utf8(save)) {
|
||||
if (save != NULL && opt != NULL && adr != NULL && adr[0]
|
||||
&& strnotempty(save) && fexist(save)) {
|
||||
const char *rsc_save = save;
|
||||
const char *rsc_fil = strrchr(fil, '/');
|
||||
int n;
|
||||
@@ -2983,11 +2985,13 @@ static void postprocess_file(httrackp *opt, const char *save, const char *adr,
|
||||
|
||||
if (!opt->state.mimehtml_created) {
|
||||
opt->state.mimefp =
|
||||
FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.mht"),
|
||||
"wb");
|
||||
(void) UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.eml"));
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.mht"),
|
||||
"wb");
|
||||
(void) unlink(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html),
|
||||
"index.eml"));
|
||||
#ifndef _WIN32
|
||||
if (symlink("index.mht",
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html),
|
||||
@@ -3628,9 +3632,6 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
|
||||
if (StringNotEmpty(from->warc_file))
|
||||
StringCopyS(to->warc_file, from->warc_file);
|
||||
to->warc_max_size = from->warc_max_size;
|
||||
to->warc_cdx = from->warc_cdx;
|
||||
to->warc_wacz = from->warc_wacz;
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
|
||||
@@ -442,10 +442,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|| (strnotempty(StringBuff(opt->path_html))))
|
||||
loops++; // do not loop once again and do not include rc file (O option exists)
|
||||
else {
|
||||
if ((!fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log"))) ||
|
||||
(argv_url > 0)) {
|
||||
if ((!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log"))) || (argv_url > 0)) {
|
||||
if (!optinclude_file(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), HTS_HTTRACKRC),
|
||||
@@ -472,12 +473,16 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
} // traiter -O
|
||||
|
||||
/* load doit.log and insert in current command line */
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/doit.log")) &&
|
||||
(argv_url <= 0)) {
|
||||
FILE *fp = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/doit.log"),
|
||||
"rb");
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/doit.log"))
|
||||
&& (argv_url <= 0)) {
|
||||
FILE *fp =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log"), "rb");
|
||||
if (fp) {
|
||||
int insert_after = 1; /* insérer après nom au début */
|
||||
|
||||
@@ -526,16 +531,23 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|
||||
/* Interrupted mirror detected */
|
||||
if (!opt->quiet) {
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"))) {
|
||||
/* Old cache */
|
||||
if ((fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"))) &&
|
||||
(fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))) {
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")))
|
||||
&&
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))) {
|
||||
if (opt->log != NULL) {
|
||||
fprintf(opt->log, "Warning!\n");
|
||||
fprintf(opt->log,
|
||||
@@ -564,82 +576,111 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
if (argv[i][1] == '-') { // --xxx
|
||||
if ((strfield2(argv[i] + 2, "clean")) || (strfield2(argv[i] + 2, "tide"))) { // nettoyer
|
||||
argv[i][1] = '\0';
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.html")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.html"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-log.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-err.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html),
|
||||
"index.html"));
|
||||
/* */
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.dat"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.ndx"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.dat"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.ndx"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.lst"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/doit.log"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"));
|
||||
RMDIR(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"));
|
||||
rmdir(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"));
|
||||
//
|
||||
} else if (strfield2(argv[i] + 2, "catchurl")) { // capture d'URL via proxy temporaire!
|
||||
argv_url = 1; // forcer a passer les parametres
|
||||
@@ -696,27 +737,29 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
#endif
|
||||
if (argv_url == 0) {
|
||||
// Présence d'un cache, que faire?..
|
||||
if ((fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"))) ||
|
||||
(fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.dat")) &&
|
||||
fexist_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))) { // il existe déja un cache
|
||||
// précédent.. renommer
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log"))) { // un cache est présent
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip")))
|
||||
||
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.dat"))
|
||||
&&
|
||||
fexist(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))
|
||||
) { // il existe déja un cache précédent.. renommer
|
||||
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/doit.log"))) { // un cache est présent
|
||||
if (x_argvblk != NULL) {
|
||||
int m;
|
||||
|
||||
// établir mode - mode cache: 1 (cache valide) 2 (cache à vérifier)
|
||||
if (fexist_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"))) { // cache prioritaire
|
||||
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-in_progress.lock"))) { // cache prioritaire
|
||||
m = 1;
|
||||
} else {
|
||||
m = 2;
|
||||
@@ -751,7 +794,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
} else { // log existe pas
|
||||
} else { // log existe pas
|
||||
HTS_PANIC_PRINTF("A cache has been found, but no command line");
|
||||
printf
|
||||
("Please launch httrack with proper parameters to reuse the cache\n");
|
||||
@@ -759,7 +802,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
} else { // aucune URL définie et pas de cache
|
||||
} else { // aucune URL définie et pas de cache
|
||||
if (opt->quiet) {
|
||||
help(argv[0], !opt->quiet);
|
||||
htsmain_free();
|
||||
@@ -774,21 +817,26 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
} else { // plus de 2 paramètres
|
||||
// un fichier log existe?
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"))) { // fichier lock?
|
||||
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"))) { // fichier lock?
|
||||
|
||||
opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
|
||||
if (opt->quiet == 0) {
|
||||
if ((fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"))) ||
|
||||
(fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat")) &&
|
||||
fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))) {
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))
|
||||
||
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))
|
||||
&&
|
||||
fexist(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))
|
||||
) {
|
||||
HT_REQUEST_START;
|
||||
HT_PRINT("There is a lock-file in the directory ");
|
||||
HT_PRINT(StringBuff(opt->path_log));
|
||||
@@ -802,19 +850,24 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.html"))) {
|
||||
} else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.html"))) {
|
||||
opt->cache = HTS_CACHE_TEST_UPDATE;
|
||||
if (opt->quiet == 0) {
|
||||
if ((fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"))) ||
|
||||
(fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat")) &&
|
||||
fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))) {
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))
|
||||
||
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))
|
||||
&&
|
||||
fexist(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))
|
||||
) {
|
||||
HT_REQUEST_START;
|
||||
HT_PRINT
|
||||
("There is an index.html and a hts-cache folder in the directory ");
|
||||
@@ -1447,13 +1500,13 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
LLint fz;
|
||||
|
||||
na++;
|
||||
fz = fsize_utf8(argv[na]);
|
||||
fz = fsize(argv[na]);
|
||||
if (fz < 0) {
|
||||
HTS_PANIC_PRINTF("File url list could not be opened");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
FILE *fp = FOPEN(argv[na], "rb");
|
||||
FILE *fp = fopen(argv[na], "rb");
|
||||
|
||||
if (fp != NULL) {
|
||||
int cl = (int) strlen(url);
|
||||
@@ -1753,34 +1806,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
return -1;
|
||||
}
|
||||
StringCopy(opt->warc_file, argv[na]);
|
||||
} else if (*(com + 1) == 's') { // --warc-max-size N: rotation
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option warc-max-size needs a blank space and a size");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
{ // reject non-numeric/negative/overflow; keep default 0
|
||||
// (single file)
|
||||
char *end;
|
||||
LLint v;
|
||||
errno = 0;
|
||||
v = strtoll(argv[na], &end, 10);
|
||||
if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
|
||||
errno != ERANGE)
|
||||
opt->warc_max_size = v;
|
||||
}
|
||||
} else if (*(com + 1) == 'c') { // --warc-cdx: sorted CDXJ index
|
||||
com++;
|
||||
opt->warc_cdx = 1;
|
||||
} else if (*(com + 1) == 'z') { // --wacz: WACZ package
|
||||
com++;
|
||||
opt->warc_wacz = 1;
|
||||
opt->warc_cdx = 1; // WACZ embeds the CDXJ index
|
||||
if (!StringNotEmpty(opt->warc_file))
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
} else { // --warc: auto-named archive under the output dir
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
@@ -1997,7 +2022,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
/*cache */ &cache, /*hash */ NULL, /*ptr */
|
||||
0, /*numero_passe */ 0, /*mime_type */
|
||||
NULL) != -1) {
|
||||
if (fexist_utf8(afs.save)) {
|
||||
if (fexist(afs.save)) {
|
||||
fprintf(stdout, "Content-location: %s\r\n",
|
||||
afs.save);
|
||||
}
|
||||
@@ -2095,16 +2120,18 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
uLong repaired = 0;
|
||||
uLong repairedBytes = 0;
|
||||
|
||||
if (fexist_utf8(fconcat(
|
||||
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"))) {
|
||||
name =
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip");
|
||||
} else if (fexist_utf8(fconcat(OPT_GET_BUFF(opt),
|
||||
OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
} else
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
name =
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip");
|
||||
@@ -2123,11 +2150,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/repair.tmp"), &repaired,
|
||||
&repairedBytes) == Z_OK) {
|
||||
UNLINK(name);
|
||||
RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/repair.zip"),
|
||||
name);
|
||||
unlink(name);
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/repair.zip"), name);
|
||||
fprintf(stderr,
|
||||
"Cache: %d bytes successfully recovered in %d entries\n",
|
||||
(int) repairedBytes, (int) repaired);
|
||||
@@ -2345,9 +2371,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
// Débuggage des en têtes
|
||||
if (_DEBUG_HEAD) {
|
||||
ioinfo = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-ioinfo.txt"),
|
||||
"wb");
|
||||
ioinfo =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-ioinfo.txt"),
|
||||
"wb");
|
||||
}
|
||||
|
||||
{
|
||||
@@ -2422,9 +2449,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
/* readme for information purpose */
|
||||
{
|
||||
FILE *fp =
|
||||
FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/readme.txt"),
|
||||
"wb");
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/readme.txt"), "wb");
|
||||
if (fp) {
|
||||
fprintf(fp, "What's in this folder?" LF);
|
||||
fprintf(fp, "" LF);
|
||||
@@ -2476,16 +2503,17 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
int i;
|
||||
|
||||
#ifdef _WIN32
|
||||
hts_mkdir_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache"));
|
||||
mkdir(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"));
|
||||
#else
|
||||
mkdir(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
|
||||
HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
fp = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/doit.log"),
|
||||
"wb");
|
||||
fp =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/doit.log"), "wb");
|
||||
if (fp) {
|
||||
for(i = 0 + 1; i < argc; i++) {
|
||||
if (((strchr(argv[i], ' ') != NULL)
|
||||
@@ -2530,7 +2558,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
}
|
||||
// petit message dans le lock
|
||||
if ((fp = FOPEN(n_lock, "wb")) != NULL) {
|
||||
if ((fp = fopen(n_lock, "wb")) != NULL) {
|
||||
int i;
|
||||
|
||||
fprintf(fp, "Mirror in progress since %s .. please wait!" LF, t);
|
||||
@@ -2690,15 +2718,16 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char *f = OPT_GET_BUFF(opt);
|
||||
|
||||
sprintf(f, "%s/%s", CACHE_REFNAME, entry->d_name);
|
||||
(void) UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), f));
|
||||
(void)
|
||||
unlink(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), f));
|
||||
}
|
||||
}
|
||||
if (dir != NULL) {
|
||||
(void) closedir(dir);
|
||||
}
|
||||
(void) RMDIR(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), CACHE_REFNAME));
|
||||
(void)
|
||||
rmdir(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), CACHE_REFNAME));
|
||||
}
|
||||
|
||||
/* Info for wrappers */
|
||||
@@ -2726,7 +2755,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
}
|
||||
// supprimer lock
|
||||
UNLINK(n_lock);
|
||||
remove(n_lock);
|
||||
}
|
||||
|
||||
if (x_argvblk)
|
||||
|
||||
@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
|
||||
configure.ac, decoupled from these). VERSION is the display form, VERSIONID
|
||||
the dotted numeric form, AFF_VERSION the short form shown in footers,
|
||||
LIB_VERSION the data/cache format generation. */
|
||||
#define HTTRACK_VERSION "3.49-14"
|
||||
#define HTTRACK_VERSIONID "3.49.14"
|
||||
#define HTTRACK_VERSION "3.49-13"
|
||||
#define HTTRACK_VERSIONID "3.49.13"
|
||||
#define HTTRACK_AFF_VERSION "3.x"
|
||||
#define HTTRACK_LIB_VERSION "2.0"
|
||||
|
||||
|
||||
@@ -412,9 +412,9 @@ void help_catchurl(const char *dest_path) {
|
||||
do {
|
||||
snprintf(dest, sizeof(dest), "%s%s%d", dest_path, "hts-post", i);
|
||||
i++;
|
||||
} while (fexist_utf8(dest));
|
||||
} while(fexist(dest));
|
||||
{
|
||||
FILE *fp = FOPEN(dest, "wb");
|
||||
FILE *fp = fopen(dest, "wb");
|
||||
|
||||
if (fp) {
|
||||
fwrite(data, strlen(data), 1, fp);
|
||||
@@ -524,6 +524,8 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %r write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
|
||||
"output name");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
@@ -593,10 +595,6 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" C create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)");
|
||||
infomsg(" k store all files in cache (not useful if files on disk)");
|
||||
infomsg(" %r write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
|
||||
"output name, --warc-max-size N rotates segments past N bytes, "
|
||||
"--warc-cdx also writes a sorted CDXJ index, --wacz packages it all "
|
||||
"as a WACZ file");
|
||||
infomsg(" %n do not re-download locally erased files");
|
||||
infomsg
|
||||
(" %v display on screen filenames downloaded (in realtime) - * %v1 short version - %v2 full animation");
|
||||
|
||||
@@ -349,13 +349,9 @@ void index_finish(const char *indexpath, int mode) {
|
||||
|
||||
// Write new file
|
||||
if (mode == 1) // TEXT
|
||||
fp = FOPEN(
|
||||
concat(catbuff, sizeof(catbuff), indexpath, "index.txt"),
|
||||
"wb");
|
||||
fp = fopen(concat(catbuff, sizeof(catbuff), indexpath, "index.txt"), "wb");
|
||||
else // HTML
|
||||
fp = FOPEN(
|
||||
concat(catbuff, sizeof(catbuff), indexpath, "sindex.html"),
|
||||
"wb");
|
||||
fp = fopen(concat(catbuff, sizeof(catbuff), indexpath, "sindex.html"), "wb");
|
||||
if (fp) {
|
||||
char current_word[KEYW_LEN + 32];
|
||||
char word[KEYW_LEN + 32];
|
||||
|
||||
112
src/htslib.c
112
src/htslib.c
@@ -6019,7 +6019,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->strip_query, "");
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->warc_file, "");
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
@@ -6431,28 +6430,21 @@ HTSEXT_API int hts_resetvar(void) {
|
||||
#ifdef _WIN32
|
||||
|
||||
typedef struct dirent dirent;
|
||||
|
||||
DIR *opendir(const char *name) {
|
||||
WIN32_FILE_ATTRIBUTE_DATA st;
|
||||
DIR *dir;
|
||||
size_t len;
|
||||
int i;
|
||||
LPWSTR wname;
|
||||
|
||||
if (name == NULL || *name == '\0') {
|
||||
errno = ENOENT;
|
||||
return NULL;
|
||||
}
|
||||
// Wide \\?\ path: no MAX_PATH cap, no CP_ACP mis-decode (#133,#630).
|
||||
wname = hts_pathToUCS2(name);
|
||||
if (wname == NULL ||
|
||||
!GetFileAttributesExW(wname, GetFileExInfoStandard, &st) ||
|
||||
(st.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) {
|
||||
freet(wname);
|
||||
if (!GetFileAttributesEx(name, GetFileExInfoStandard, &st)
|
||||
|| (st.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) {
|
||||
errno = ENOENT;
|
||||
return NULL;
|
||||
}
|
||||
freet(wname);
|
||||
dir = calloc(sizeof(DIR), 1);
|
||||
if (dir == NULL) {
|
||||
errno = ENOMEM;
|
||||
@@ -6472,29 +6464,19 @@ DIR *opendir(const char *name) {
|
||||
}
|
||||
|
||||
struct dirent *readdir(DIR * dir) {
|
||||
WIN32_FIND_DATAW find;
|
||||
WIN32_FIND_DATAA find;
|
||||
|
||||
if (dir->h == INVALID_HANDLE_VALUE) {
|
||||
// \\?\-prefix so a long/non-ASCII directory enumerates instead of ENOENT.
|
||||
LPWSTR wname = hts_pathToUCS2(dir->name);
|
||||
|
||||
dir->h =
|
||||
wname != NULL ? FindFirstFileW(wname, &find) : INVALID_HANDLE_VALUE;
|
||||
freet(wname);
|
||||
dir->h = FindFirstFileA(dir->name, &find);
|
||||
} else {
|
||||
if (!FindNextFileW(dir->h, &find)) {
|
||||
if (!FindNextFile(dir->h, &find)) {
|
||||
FindClose(dir->h);
|
||||
dir->h = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
}
|
||||
if (dir->h != INVALID_HANDLE_VALUE) {
|
||||
char *u = hts_convertUCS2StringToUTF8(find.cFileName, -1);
|
||||
|
||||
dir->entry.d_name[0] = 0;
|
||||
if (u != NULL) {
|
||||
strncat(dir->entry.d_name, u, HTS_DIRENT_SIZE - 1);
|
||||
freet(u);
|
||||
}
|
||||
strncat(dir->entry.d_name, find.cFileName, HTS_DIRENT_SIZE - 1);
|
||||
return &dir->entry;
|
||||
}
|
||||
errno = ENOENT;
|
||||
@@ -6527,61 +6509,9 @@ static void copyWchar(LPWSTR dest, const char *src) {
|
||||
dest[i] = '\0';
|
||||
}
|
||||
|
||||
/* UTF-8 path -> UCS-2 for the _w* file APIs. At/above HTS_WIN_LONGPATH_MIN,
|
||||
\\?\-prefix it via GetFullPathNameW to clear MAX_PATH (#133); else unchanged.
|
||||
Any prefixing failure falls back to the plain converted path. */
|
||||
#define HTS_WIN_LONGPATH_MIN 240 /* stay clear of MAX_PATH (260) */
|
||||
|
||||
LPWSTR hts_pathToUCS2(const char *path) {
|
||||
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);
|
||||
|
||||
if (wpath == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
const size_t len = wcslen(wpath);
|
||||
// Already "\\?\" or "\\.\": don't re-prefix.
|
||||
const int verbatim = len >= 4 && wpath[0] == L'\\' && wpath[1] == L'\\' &&
|
||||
(wpath[2] == L'?' || wpath[2] == L'.') &&
|
||||
wpath[3] == L'\\';
|
||||
|
||||
if (len < HTS_WIN_LONGPATH_MIN || verbatim) {
|
||||
return wpath;
|
||||
}
|
||||
|
||||
const DWORD need = GetFullPathNameW(wpath, 0, NULL, NULL); /* incl NUL */
|
||||
LPWSTR full = need != 0 ? malloct((size_t) need * sizeof(WCHAR)) : NULL;
|
||||
|
||||
if (full == NULL) {
|
||||
return wpath; /* fall back to the plain path */
|
||||
}
|
||||
const DWORD written = GetFullPathNameW(wpath, need, full, NULL);
|
||||
|
||||
if (written == 0 || written >= need || full[0] == L'\0') {
|
||||
freet(full);
|
||||
return wpath;
|
||||
}
|
||||
|
||||
const int isUNC = full[0] == L'\\' && full[1] == L'\\';
|
||||
// UNC "\\srv\share" -> "\\?\UNC\srv\share": the prefix subsumes the "\\".
|
||||
const WCHAR *const pfx = isUNC ? L"\\\\?\\UNC\\" : L"\\\\?\\";
|
||||
const WCHAR *const body = isUNC ? full + 2 : full;
|
||||
const size_t pfxLen = wcslen(pfx), bodyLen = wcslen(body);
|
||||
LPWSTR out = malloct((pfxLen + bodyLen + 1) * sizeof(WCHAR));
|
||||
|
||||
if (out == NULL) {
|
||||
freet(full);
|
||||
return wpath;
|
||||
}
|
||||
memcpybuff(out, pfx, pfxLen * sizeof(WCHAR));
|
||||
memcpybuff(out + pfxLen, body, (bodyLen + 1) * sizeof(WCHAR));
|
||||
freet(full);
|
||||
freet(wpath);
|
||||
return out;
|
||||
}
|
||||
|
||||
FILE *hts_fopen_utf8(const char *path, const char *mode) {
|
||||
WCHAR wmode[32];
|
||||
LPWSTR wpath = hts_pathToUCS2(path);
|
||||
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);
|
||||
|
||||
assertf(strlen(mode) < sizeof(wmode) / sizeof(WCHAR));
|
||||
copyWchar(wmode, mode);
|
||||
@@ -6597,7 +6527,7 @@ FILE *hts_fopen_utf8(const char *path, const char *mode) {
|
||||
}
|
||||
|
||||
int hts_stat_utf8(const char *path, STRUCT_STAT * buf) {
|
||||
LPWSTR wpath = hts_pathToUCS2(path);
|
||||
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);
|
||||
|
||||
if (wpath != NULL) {
|
||||
const int result = _wstat64(wpath, buf);
|
||||
@@ -6611,7 +6541,7 @@ int hts_stat_utf8(const char *path, STRUCT_STAT * buf) {
|
||||
}
|
||||
|
||||
int hts_unlink_utf8(const char *path) {
|
||||
LPWSTR wpath = hts_pathToUCS2(path);
|
||||
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);
|
||||
|
||||
if (wpath != NULL) {
|
||||
const int result = _wunlink(wpath);
|
||||
@@ -6625,8 +6555,10 @@ int hts_unlink_utf8(const char *path) {
|
||||
}
|
||||
|
||||
int hts_rename_utf8(const char *oldpath, const char *newpath) {
|
||||
LPWSTR woldpath = hts_pathToUCS2(oldpath);
|
||||
LPWSTR wnewpath = hts_pathToUCS2(newpath);
|
||||
LPWSTR woldpath =
|
||||
hts_convertUTF8StringToUCS2(oldpath, (int) strlen(oldpath), NULL);
|
||||
LPWSTR wnewpath =
|
||||
hts_convertUTF8StringToUCS2(newpath, (int) strlen(newpath), NULL);
|
||||
if (woldpath != NULL && wnewpath != NULL) {
|
||||
const int result = _wrename(woldpath, wnewpath);
|
||||
|
||||
@@ -6643,22 +6575,8 @@ int hts_rename_utf8(const char *oldpath, const char *newpath) {
|
||||
}
|
||||
}
|
||||
|
||||
int hts_rmdir_utf8(const char *path) {
|
||||
LPWSTR wpath = hts_pathToUCS2(path);
|
||||
|
||||
if (wpath != NULL) {
|
||||
const int result = _wrmdir(wpath);
|
||||
|
||||
free(wpath);
|
||||
return result;
|
||||
} else {
|
||||
// Fallback on conversion error.
|
||||
return _rmdir(path);
|
||||
}
|
||||
}
|
||||
|
||||
int hts_mkdir_utf8(const char *path) {
|
||||
LPWSTR wpath = hts_pathToUCS2(path);
|
||||
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);
|
||||
|
||||
if (wpath != NULL) {
|
||||
const int result = _wmkdir(wpath);
|
||||
@@ -6673,7 +6591,7 @@ int hts_mkdir_utf8(const char *path) {
|
||||
|
||||
HTSEXT_API int hts_utime_utf8(const char *path, const STRUCT_UTIMBUF * times) {
|
||||
STRUCT_UTIMBUF mtimes = *times;
|
||||
LPWSTR wpath = hts_pathToUCS2(path);
|
||||
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);
|
||||
|
||||
if (wpath != NULL) {
|
||||
const int result = _wutime(wpath, &mtimes);
|
||||
|
||||
@@ -609,9 +609,7 @@ static HTS_UNUSED size_t llint_to_size_t(LLint o) {
|
||||
|
||||
/* dirent() compatibility */
|
||||
#ifdef _WIN32
|
||||
/* Holds a UTF-8 d_name: MAX_PATH (260) UTF-16 units expand to <=3 bytes each.
|
||||
Windows-only struct, ABI free to break. */
|
||||
#define HTS_DIRENT_SIZE 1024
|
||||
#define HTS_DIRENT_SIZE 256
|
||||
struct dirent {
|
||||
ino_t d_ino; /* ignored */
|
||||
off_t d_off; /* ignored */
|
||||
|
||||
@@ -47,7 +47,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
#include <ctype.h>
|
||||
#include <limits.h>
|
||||
|
||||
#define ADD_STANDARD_PATH \
|
||||
{ /* ajout nom */\
|
||||
@@ -1476,37 +1475,15 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
sizeof(afs->save) - (size_t) (lastDot - afs->save));
|
||||
}
|
||||
}
|
||||
// Cap the save path: the final parent+name is copied into a fixed buffer that
|
||||
// aborts() on overflow (htssafe.h), so clamp every ceiling to fit it.
|
||||
#define HTS_SAVE_BUFSIZE (HTS_URLMAXSIZE * 2) /* sizeof(afs->save) */
|
||||
#define HTS_PATH_TAIL_RESERVE 64 /* collision suffix + ".delayed" + NUL */
|
||||
#ifdef _WIN32
|
||||
// MAX_PATH minus 8.3 headroom (MSDN) minus the ".delayed" marker; raising it
|
||||
// needs the engine to "\\?\"-prefix its paths, which is separate work.
|
||||
#define HTS_MAX_PATH_LEN (260 - 12 - 12)
|
||||
#define MAX_SEG_LEN 48
|
||||
#else
|
||||
// #133: use the platform's own PATH_MAX/NAME_MAX (Linux/Android 4096, macOS
|
||||
// 1024) rather than the far smaller Windows MAX_PATH.
|
||||
#ifdef PATH_MAX
|
||||
#define HTS_PATH_MAX_ PATH_MAX
|
||||
#else
|
||||
#define HTS_PATH_MAX_ 1024
|
||||
#endif
|
||||
#ifdef NAME_MAX
|
||||
#define HTS_NAME_MAX_ NAME_MAX
|
||||
#else
|
||||
#define HTS_NAME_MAX_ 255
|
||||
#endif
|
||||
#define HTS_MAX_PATH_LEN \
|
||||
((HTS_PATH_MAX_ - HTS_PATH_TAIL_RESERVE) < \
|
||||
(HTS_SAVE_BUFSIZE - HTS_PATH_TAIL_RESERVE) \
|
||||
? (HTS_PATH_MAX_ - HTS_PATH_TAIL_RESERVE) \
|
||||
: (HTS_SAVE_BUFSIZE - HTS_PATH_TAIL_RESERVE))
|
||||
#define MAX_SEG_LEN (HTS_NAME_MAX_ > 64 ? HTS_NAME_MAX_ - 16 : HTS_NAME_MAX_)
|
||||
#endif
|
||||
// enforce 260-character path limit before inserting destination path
|
||||
// note: 12 characters at least for WIN32, and 12 for ".99.delayed"
|
||||
// (MSDN) "When using an API to create a directory, the specified path
|
||||
// cannot be so long that you cannot append an 8.3 file name
|
||||
// (that is, the directory name cannot exceed MAX_PATH minus 12)."
|
||||
#define HTS_MAX_PATH_LEN ( 260 - 12 - 12 )
|
||||
#define MIN_LAST_SEG_RESERVE 12
|
||||
#define MAX_LAST_SEG_RESERVE 24
|
||||
#define MAX_SEG_LEN 48
|
||||
if (hts_stringLengthUTF8(afs->save) +
|
||||
hts_stringLengthUTF8(StringBuff(opt->path_html_utf8)) >=
|
||||
HTS_MAX_PATH_LEN) {
|
||||
@@ -1516,7 +1493,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (wsave != NULL) {
|
||||
const size_t parentLen =
|
||||
hts_stringLengthUTF8(StringBuff(opt->path_html_utf8));
|
||||
// parent path length is not insane (otherwise, ignore and pick 200 as
|
||||
// parent path length is not insane (otherwise, ignore and pick 200 as
|
||||
// suffix length)
|
||||
const size_t maxLen =
|
||||
parentLen <
|
||||
@@ -1607,37 +1584,9 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// Re-check again ending space or dot after cut (see bug #5)
|
||||
cleanEndingSpaceOrDot(afs->save);
|
||||
}
|
||||
// The cut above counts UTF-8 codepoints, but parent+name lands in a fixed
|
||||
// byte buffer that aborts() on overflow (htssafe.h). A multibyte name can
|
||||
// pass the codepoint cap yet overflow in bytes, so hard-cut on a codepoint
|
||||
// boundary to keep parent+name inside the buffer regardless (#133).
|
||||
{
|
||||
const size_t parentBytes = strlen(StringBuff(opt->path_html_utf8));
|
||||
const size_t cap = HTS_SAVE_BUFSIZE - HTS_PATH_TAIL_RESERVE;
|
||||
// Shrink only the name. A parent that alone fills the buffer is left to the
|
||||
// existing prepend abort, not collapsed to an empty name that would collide
|
||||
// across URLs and overrun the unbounded collision-suffix sprintf.
|
||||
if (parentBytes < cap) {
|
||||
size_t budget = cap - parentBytes;
|
||||
if (strlen(afs->save) > budget) {
|
||||
while (budget > 0 && ((unsigned char) afs->save[budget] & 0xC0) == 0x80)
|
||||
budget--; // back off a continuation byte, never split a char
|
||||
afs->save[budget] = '\0';
|
||||
cleanEndingSpaceOrDot(afs->save);
|
||||
}
|
||||
}
|
||||
}
|
||||
#undef MAX_UTF8_SEQ_CHARS
|
||||
#undef MIN_LAST_SEG_RESERVE
|
||||
#undef MAX_LAST_SEG_RESERVE
|
||||
#undef MAX_SEG_LEN
|
||||
#undef HTS_MAX_PATH_LEN
|
||||
#undef HTS_PATH_TAIL_RESERVE
|
||||
#undef HTS_SAVE_BUFSIZE
|
||||
#ifndef _WIN32
|
||||
#undef HTS_PATH_MAX_
|
||||
#undef HTS_NAME_MAX_
|
||||
#endif
|
||||
|
||||
// chemin primaire éventuel A METTRE AVANT
|
||||
if (strnotempty(StringBuff(opt->path_html_utf8))) {
|
||||
|
||||
11
src/htsopt.h
11
src/htsopt.h
@@ -541,12 +541,6 @@ struct httrackp {
|
||||
and exit without crawling */
|
||||
String warc_file; /**< WARC output: WARC_AUTONAME for --warc, or the
|
||||
--warc-file basename (appended at the tail: ABI) */
|
||||
LLint warc_max_size; /**< --warc-max-size: rotate the archive past this many
|
||||
bytes (<=0: single file). Tail: ABI */
|
||||
hts_boolean warc_cdx; /**< --warc-cdx: write a sorted CDXJ index next to the
|
||||
archive. Tail: ABI */
|
||||
hts_boolean warc_wacz; /**< --wacz: package archive+index+pages as a WACZ zip
|
||||
(implies --warc + --warc-cdx). Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
@@ -673,11 +667,6 @@ struct htsblk {
|
||||
char *warc_reqhdr; /**< stashed raw request header block for WARC (or NULL) */
|
||||
char *
|
||||
warc_resphdr; /**< stashed raw response header block for WARC (or NULL) */
|
||||
int warc_truncated; /**< WARC-Truncated reason for a cap-truncated body
|
||||
(WARC_TRUNC_*, 0=none). Tail: ABI */
|
||||
char *warc_rawpath; /**< verbatim WARC: spooled compressed body path, or NULL
|
||||
(owns the file; unlinked on free). Tail: ABI */
|
||||
LLint warc_rawsize; /**< byte length of warc_rawpath. Tail: ABI */
|
||||
/*char digest[32+2]; // md5 digest generated by the engine ("" if none) */
|
||||
};
|
||||
|
||||
|
||||
@@ -3656,7 +3656,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
!ref_existed;
|
||||
if (fexist_utf8(heap(ptr)->sav)) {
|
||||
had_partial = 1;
|
||||
UNLINK(heap(ptr)->sav);
|
||||
remove(heap(ptr)->sav);
|
||||
}
|
||||
|
||||
// Re-get once, only if a partial existed and both Range triggers are
|
||||
@@ -3862,13 +3862,18 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
|
||||
int do_pause = 0;
|
||||
|
||||
// user pause lockfile : create hts-paused.lock --> HTTrack will be paused
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-stop.lock"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-stop.lock"))) {
|
||||
// remove lockfile
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-stop.lock"));
|
||||
if (!fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-stop.lock"))) {
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-stop.lock"));
|
||||
if (!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-stop.lock"))) {
|
||||
do_pause = 1;
|
||||
}
|
||||
}
|
||||
@@ -3897,9 +3902,11 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
|
||||
}
|
||||
}
|
||||
{
|
||||
FILE *fp = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-paused.lock"),
|
||||
"wb");
|
||||
FILE *fp =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-paused.lock"), "wb");
|
||||
if (fp) {
|
||||
fspc(NULL, fp, "info"); // dater
|
||||
fprintf(fp,
|
||||
@@ -4215,17 +4222,18 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
|
||||
int a = 0;
|
||||
|
||||
*stre->last_info_shell_ = tl;
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-autopsy"))) { // débuggage: teste si le
|
||||
// robot est vivant
|
||||
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-autopsy"))) { // débuggage: teste si le robot est vivant
|
||||
// (oui je sais un robot vivant.. mais bon.. il a le droit de vivre lui aussi)
|
||||
// (libérons les robots esclaves de l'internet!)
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-autopsy"));
|
||||
fp = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-isalive"),
|
||||
"wb");
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-autopsy"));
|
||||
fp =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-isalive"), "wb");
|
||||
a = 1;
|
||||
}
|
||||
if ((*stre->info_shell_) || a) {
|
||||
|
||||
1255
src/htsselftest.c
1255
src/htsselftest.c
File diff suppressed because it is too large
Load Diff
@@ -1027,17 +1027,6 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
|
||||
freet(category);
|
||||
category = NULL;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
/* category is ANSI-codepage, doc is utf-8: convert (#216) */
|
||||
else {
|
||||
char *cat_utf8 = hts_convertStringSystemToUTF8(
|
||||
category, strlen(category));
|
||||
if (cat_utf8 != NULL) {
|
||||
freet(category);
|
||||
category = cat_utf8;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
if (category == NULL) {
|
||||
@@ -1055,18 +1044,7 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
|
||||
oldchain->next = chain;
|
||||
}
|
||||
chain->next = NULL;
|
||||
#ifdef _WIN32
|
||||
/* name is ANSI-codepage, doc is utf-8: convert (#216) */
|
||||
{
|
||||
const char *const name = hts_findgetname(h);
|
||||
char *name_utf8 =
|
||||
hts_convertStringSystemToUTF8(name, strlen(name));
|
||||
strcpybuff(chain->name, name_utf8 != NULL ? name_utf8 : name);
|
||||
freet(name_utf8);
|
||||
}
|
||||
#else
|
||||
strcpybuff(chain->name, hts_findgetname(h));
|
||||
#endif
|
||||
chain->category = category;
|
||||
chain->level = level;
|
||||
}
|
||||
|
||||
1057
src/htswarc.c
1057
src/htswarc.c
File diff suppressed because it is too large
Load Diff
@@ -45,16 +45,6 @@ extern "C" {
|
||||
under the project's output directory at open time. */
|
||||
#define WARC_AUTONAME "\001auto"
|
||||
|
||||
/* htsblk.warc_truncated / WARC-Truncated reason tokens (ISO 28500 sec 5.13).
|
||||
A cap-truncated body is still archived, tagged with why it was cut short. */
|
||||
#define WARC_TRUNC_NONE 0
|
||||
#define WARC_TRUNC_LENGTH 1 /* hit a size cap (-M mirror / -m per-file) */
|
||||
#define WARC_TRUNC_TIME 2 /* hit the mirror time cap (-E/--max-time) */
|
||||
#define WARC_TRUNC_DISCONNECT 3 /* connection dropped mid-body */
|
||||
|
||||
/* WARC-Truncated token for a warc_truncated code, or NULL for none. */
|
||||
const char *warc_truncated_reason(int code);
|
||||
|
||||
typedef struct warc_writer warc_writer;
|
||||
|
||||
/* Stash the raw request header block (bstr.buffer) on r for the later WARC
|
||||
@@ -68,13 +58,6 @@ void warc_stash_response(htsblk *r, const char *resphdr);
|
||||
/* Free both stashed header blocks (idempotent, NULL-safe). */
|
||||
void warc_free_request(htsblk *r);
|
||||
|
||||
/* Adopt the de-chunked compressed spool at tmpfile_path onto
|
||||
r->warc_rawpath/warc_rawsize (strdupt; frees any prior) so the WARC record
|
||||
stores the body verbatim. No-op leaving warc_rawpath NULL when tmpfile_path
|
||||
is empty or the spool is missing/empty (the record then stores the decoded
|
||||
in-memory/on-disk body instead). */
|
||||
void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path);
|
||||
|
||||
/* Emit the request + response (or revisit) records for one finished
|
||||
transaction. Lazily opens the writer into opt->state.warc; a no-op (logged
|
||||
once) if the archive cannot be created. */
|
||||
@@ -93,10 +76,6 @@ warc_writer *warc_open(httrackp *opt, const char *path);
|
||||
/* Flush, close and free the writer (NULL-safe). */
|
||||
void warc_close(warc_writer *w);
|
||||
|
||||
/* SURT-canonicalize url into out[outsz] (the CDXJ sort key). Returns 0 on
|
||||
success, -1 on error or truncation. Exposed for the -#test=warc-surt test. */
|
||||
int warc_surt(const char *url, char *out, size_t outsz);
|
||||
|
||||
/* Write one transaction's request + response (or revisit) records.
|
||||
target_uri: absolute URL fetched.
|
||||
ip: numeric peer IP, or NULL/"" to omit.
|
||||
@@ -105,24 +84,12 @@ int warc_surt(const char *url, char *out, size_t outsz);
|
||||
body/body_len: decoded in-memory body, or NULL when on disk.
|
||||
body_path: file re-read for the body when body==NULL (may be NULL).
|
||||
is_update_unchanged: nonzero for a 304 server-not-modified revisit.
|
||||
truncated: a WARC_TRUNC_* reason to tag a cap-truncated body, else 0.
|
||||
The body is stored verbatim: Content-Encoding is kept and Content-Length set
|
||||
to body_len, so body/body_len must be the as-received (coded) bytes.
|
||||
Returns 0 on success, -1 on error. */
|
||||
int warc_write_transaction(warc_writer *w, const char *target_uri,
|
||||
const char *ip, const char *req_hdr,
|
||||
const char *resp_hdr, const char *body,
|
||||
size_t body_len, const char *body_path,
|
||||
int statuscode, int is_update_unchanged,
|
||||
int truncated);
|
||||
|
||||
/* Write one non-HTTP capture as a single WARC 'resource' record: the block is
|
||||
the raw payload (no HTTP envelope), Content-Type is the payload's own MIME.
|
||||
Used for ftp:// transfers. truncated is a WARC_TRUNC_* reason or 0.
|
||||
Returns 0 on success, -1 on error. */
|
||||
int warc_write_resource(warc_writer *w, const char *target_uri, const char *ip,
|
||||
const char *content_type, const char *body,
|
||||
size_t body_len, const char *body_path, int truncated);
|
||||
int statuscode, int is_update_unchanged);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@@ -758,9 +758,6 @@ HTSEXT_API int hts_rename_utf8(const char *oldpath, const char *newpath);
|
||||
|
||||
HTSEXT_API int hts_mkdir_utf8(const char *pathname);
|
||||
|
||||
#define RMDIR hts_rmdir_utf8
|
||||
HTSEXT_API int hts_rmdir_utf8(const char *pathname);
|
||||
|
||||
#define UTIME(A, B) hts_utime_utf8(A, B)
|
||||
|
||||
typedef struct _utimbuf STRUCT_UTIMBUF;
|
||||
@@ -775,7 +772,6 @@ typedef struct stat STRUCT_STAT;
|
||||
#define UNLINK unlink
|
||||
#define RENAME rename
|
||||
#define MKDIR(F) mkdir(F, HTS_ACCESS_FOLDER)
|
||||
#define RMDIR rmdir
|
||||
|
||||
typedef struct utimbuf STRUCT_UTIMBUF;
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
#endif
|
||||
|
||||
VS_VERSION_INFO VERSIONINFO
|
||||
FILEVERSION 3, 49, 14, 0
|
||||
PRODUCTVERSION 3, 49, 14, 0
|
||||
FILEVERSION 3, 49, 13, 0
|
||||
PRODUCTVERSION 3, 49, 13, 0
|
||||
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
|
||||
#ifdef _DEBUG
|
||||
FILEFLAGS VS_FF_DEBUG
|
||||
@@ -35,12 +35,12 @@ BEGIN
|
||||
BEGIN
|
||||
VALUE "CompanyName", "Xavier Roche"
|
||||
VALUE "FileDescription", VER_FILE_DESCRIPTION
|
||||
VALUE "FileVersion", "3.49.14"
|
||||
VALUE "FileVersion", "3.49.13"
|
||||
VALUE "InternalName", VER_ORIGINAL_FILENAME
|
||||
VALUE "LegalCopyright", "Copyright (C) 1998-2026 Xavier Roche and other contributors. GNU GPL v3 or later."
|
||||
VALUE "OriginalFilename", VER_ORIGINAL_FILENAME
|
||||
VALUE "ProductName", "HTTrack Website Copier"
|
||||
VALUE "ProductVersion", "3.49-14"
|
||||
VALUE "ProductVersion", "3.49-13"
|
||||
END
|
||||
END
|
||||
BLOCK "VarFileInfo"
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Drives -#test=cookieimport: load a cookie jar (and, on Windows, copied IE
|
||||
# cookies *@*.txt) from a long, non-ASCII folder through the UTF-8/long-path
|
||||
# file wrappers (#133,#630).
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=cookieimport "$dir" | grep -q "cookieimport:.*OK"
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Drives -#test=direnum: enumerate a long+non-ASCII directory through the
|
||||
# opendir/readdir wrappers, checking each child round-trips as UTF-8 (#133,#630).
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=direnum "$dir" | grep -q "direnum:.*OK"
|
||||
@@ -8,14 +8,9 @@ set -euo pipefail
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
rc=0
|
||||
out=$(httrack -#test=fsize "$dir") || rc=$?
|
||||
out=$(httrack -#test=fsize "$dir")
|
||||
echo "$out"
|
||||
|
||||
# 77 = platform can't host the 5GB probe (EFBIG); skip, don't fail.
|
||||
[ "$rc" = 77 ] && exit 77
|
||||
[ "$rc" = 0 ] || exit "$rc"
|
||||
|
||||
want="fsize: width=8,8 size=5368709120,5368709120 psize=5368709120 absent=-1"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: unexpected size report (want '$want')"
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Drives -#test=longpath: a >MAX_PATH round trip exercising hts_pathToUCS2's
|
||||
# \\?\ prefixing on Windows (#133); a positive control on POSIX.
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=longpath "$dir" | grep -q "longpath:.*OK"
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Drives -#test=mirrorio: rounds a file through a path that is both long
|
||||
# (>MAX_PATH) and non-ASCII, exercising the mirror I/O wrappers the engine's
|
||||
# raw file ops now route to on Windows (#133, #630). Positive control on POSIX.
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=mirrorio "$dir" | grep -q "mirrorio:.*OK"
|
||||
@@ -149,19 +149,10 @@ full '/page.php?id=3&sid=42' 'text/html' '/dev/null/www.example.com/page475b.htm
|
||||
strip=sid 'prior=www.example.com|/page.php?id=4|/dev/null/www.example.com/PAGE-PRIOR.html'
|
||||
|
||||
# Hostile fils stay rooted under the mirror: ../ (raw or %2e-encoded) drops out,
|
||||
# control characters become spaces.
|
||||
# control characters become spaces, oversized names cap at 210 chars (the cap
|
||||
# can chop the extension off entirely).
|
||||
full '/../../etc/passwd' 'text/html' '/dev/null/www.example.com///etc/passwd.html'
|
||||
full '/%2e%2e/%2e%2e/etc/passwd' 'text/html' '/dev/null/www.example.com///etc/passwd.html'
|
||||
full '/x.php' 'application/pdf' '/dev/null/www.example.com///evil.exe' 'cdispo=../../evil.exe'
|
||||
name $'/evil\rname\t.php' 'text/html' 'evil name .html'
|
||||
|
||||
# #133: oversized names are capped only on Windows (MAX_PATH, chopping the
|
||||
# extension); elsewhere the platform PATH_MAX leaves a 300-char name whole.
|
||||
case "$(uname -s 2>/dev/null)" in
|
||||
MINGW* | MSYS* | CYGWIN*)
|
||||
name "/$(printf 'a%.0s' {1..300}).php" 'text/html' "$(printf 'a%.0s' {1..210})"
|
||||
;;
|
||||
*)
|
||||
name "/$(printf 'a%.0s' {1..300}).php" 'text/html' "$(printf 'a%.0s' {1..300}).html"
|
||||
;;
|
||||
esac
|
||||
name "/$(printf 'a%.0s' {1..300}).php" 'text/html' "$(printf 'a%.0s' {1..210})"
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# SURT canonicalization of the CDXJ sort key (--warc-cdx). Pure string work,
|
||||
# so it runs under the MSan-instrumented 01_engine glob.
|
||||
httrack -O /dev/null -#test=warc-surt | grep -q "warc-surt: OK"
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --warc-cdx CDXJ index over synthetic transactions: sorted, one line per
|
||||
# response/revisit/resource, each offset/length inflates to the right member.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc-cdx "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --wacz packaging over synthetic transactions: the WACZ unzips in-process to
|
||||
# the fixed layout, every entry is ZIP STORE, each datapackage sha256 recomputes
|
||||
# from the stored bytes, and the digest chains datapackage.json. WACZ needs the
|
||||
# OpenSSL SHA-256 digests, so the self-test is absent on non-OpenSSL builds.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
if ! "$httrack_bin" -#test 2>&1 | grep -q '^ warc-wacz'; then
|
||||
echo "warc-wacz self-test unavailable (build without OpenSSL); skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc-wacz "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
@@ -4,19 +4,16 @@
|
||||
set -euo pipefail
|
||||
|
||||
# WARC/1.1 writer self-test: framing, Content-Length, gzip members, round-trip,
|
||||
# revisit dedup, plus v1.1 WARC-Truncated, ftp resource records, and
|
||||
# --warc-max-size rotation, over synthetic transactions.
|
||||
# and identical-payload-digest revisit dedup, over synthetic transactions.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
for t in warc warc-trunc warc-ftp warc-rotate warc-verbatim; do
|
||||
out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
done
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
|
||||
@@ -3,68 +3,30 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# #623: url_savename shortens an over-ceiling path by cutting the tail of the
|
||||
# last segment, where the mandatory ".<id>.delayed" placeholder lives. A cut
|
||||
# marker fails IS_DELAYED_EXT, so back_delayed_rename never renames the file and
|
||||
# the download is lost. The marker must survive the cut.
|
||||
#
|
||||
# The ceiling is the Windows MAX_PATH on Windows but the (far larger) save
|
||||
# buffer elsewhere (#133). Each CLI arg is capped at HTS_CDLMAXSIZE (1024), so
|
||||
# on POSIX we lengthen the output dir as well to overrun the ceiling.
|
||||
# #623: url_savename enforces the 236-char path ceiling by cutting the tail of
|
||||
# the last segment, where the mandatory ".delayed" placeholder marker lives. A
|
||||
# cut marker fails IS_DELAYED_EXT, so back_delayed_rename never renames the file
|
||||
# to its final name and the download is lost. The marker must survive the cut.
|
||||
|
||||
# statuscode=302 status=-1 = a redirect answer still downloading: no type is
|
||||
# resolved, so the name gets a ".<id>.delayed" placeholder (see 01_engine-savename).
|
||||
CEIL=236
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8}) # 128 hex chars
|
||||
|
||||
# engine ceiling is the Windows MAX_PATH on Windows, the save buffer elsewhere
|
||||
case "$(uname -s 2>/dev/null)" in
|
||||
MINGW* | MSYS* | CYGWIN*)
|
||||
ceil=236
|
||||
outdir=/dev/null
|
||||
deep="/d1$(printf 'a%.0s' {1..40})/d2$(printf 'b%.0s' {1..40})"
|
||||
deep="$deep/d3$(printf 'c%.0s' {1..40})/d4$(printf 'd%.0s' {1..40})"
|
||||
long="$deep/$(printf 'z%.0s' {1..90})"
|
||||
hexpath="$deep/$hexseg"
|
||||
;;
|
||||
*)
|
||||
# overrun the save-buffer ceiling with a long dir + long name
|
||||
ceil=$((1024 * 2 - 64)) # HTS_URLMAXSIZE*2 - HTS_PATH_TAIL_RESERVE
|
||||
outdir="$scratch/$(printf 'D%.0s' $(seq 1 $((1000 - ${#scratch}))))"
|
||||
long="/d1/$(printf 'z%.0s' $(seq 1 985))"
|
||||
hexpath="/d1/$(printf 'y%.0s' $(seq 1 850))/$hexseg"
|
||||
;;
|
||||
esac
|
||||
|
||||
run() {
|
||||
"$httrack_bin" -O "$outdir" -#test=savename "$@" | sed -n 's/^savename: //p'
|
||||
"$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
|
||||
}
|
||||
|
||||
# untruncated "outdir/host/name" is longer than the ceiling, so truncation must
|
||||
# fire; out then sits at or under it. Together these keep the marker check below
|
||||
# non-vacuous.
|
||||
check_truncated() {
|
||||
local out=$1 fil=$2
|
||||
local untrunc=$((${#outdir} + 1 + 15 + ${#fil})) # host = www.example.com
|
||||
test "$untrunc" -gt "$ceil" ||
|
||||
{
|
||||
echo "FAIL: input too short to truncate ($untrunc <= $ceil)"
|
||||
exit 1
|
||||
}
|
||||
test "${#out}" -le "$ceil" ||
|
||||
{
|
||||
echo "FAIL: truncated name ${#out} > $ceil ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
# A deep path ending in a long segment that overruns the ceiling.
|
||||
deep="/d1$(printf 'a%.0s' {1..40})/d2$(printf 'b%.0s' {1..40})"
|
||||
deep="$deep/d3$(printf 'c%.0s' {1..40})/d4$(printf 'd%.0s' {1..40})"
|
||||
long="$deep/$(printf 'z%.0s' {1..90})"
|
||||
|
||||
# statuscode=302 status=-1 = a redirect still downloading: no type is resolved,
|
||||
# so the name gets a ".<id>.delayed" placeholder (see 01_engine-savename).
|
||||
long_delayed="$long.ext"
|
||||
out="$(run "$long_delayed" text/html statuscode=302 status=-1)"
|
||||
check_truncated "$out" "$long_delayed"
|
||||
out="$(run "$long" text/html statuscode=302 status=-1)"
|
||||
case "$out" in
|
||||
*.delayed) ;;
|
||||
*)
|
||||
@@ -72,11 +34,16 @@ case "$out" in
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
test "${#out}" -le "$CEIL" || {
|
||||
echo "FAIL: truncated name ${#out} > $CEIL ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# #133-style hashed name: an all-hex last segment must keep the marker too (the
|
||||
# ".<id>." tag is not mistaken for part of the hash).
|
||||
hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8})
|
||||
hexpath="/d1$(printf 'x%.0s' {1..30})/d2$(printf 'y%.0s' {1..30})/$hexseg"
|
||||
out="$(run "$hexpath" text/html statuscode=302 status=-1)"
|
||||
check_truncated "$out" "$hexpath"
|
||||
case "$out" in
|
||||
*.delayed) ;;
|
||||
*)
|
||||
@@ -84,10 +51,17 @@ case "$out" in
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
test "${#out}" -le "$CEIL" || {
|
||||
echo "FAIL: hashed name ${#out} > $CEIL ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# A non-delayed name of the same shape still truncates, with no marker to keep.
|
||||
out="$(run "$long_delayed.html" text/html)"
|
||||
check_truncated "$out" "$long_delayed.html"
|
||||
out="$(run "$long.html" text/html)"
|
||||
test "${#out}" -le "$CEIL" || {
|
||||
echo "FAIL: non-delayed name ${#out} > $CEIL ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
case "$out" in
|
||||
*.delayed)
|
||||
echo "FAIL: non-delayed name grew a .delayed marker: '$out'"
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A --warc crawl stores compressed bodies verbatim (the default): the response
|
||||
# record keeps Content-Encoding: gzip and its stored bytes inflate back to the
|
||||
# served page. The validator's --verbatim mode is the differential gate:
|
||||
# inflate(stored) must equal the decoded body the server compressed.
|
||||
#
|
||||
# Two fixtures cover both adoption branches of back_finalize: page.html (text/html)
|
||||
# takes the in-memory branch; data.bin (application/octet-stream) is streamed to
|
||||
# disk, so it exercises the is_write direct-to-disk spool adoption.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# decoded bodies served (gzip-coded) by route_warcgz_page and route_warcgz_data.
|
||||
export WARC_VALIDATE_BODY="warcgz/page.html=3c68746d6c3e3c626f64793e766572626174696d20677a6970207061676520666f72205741524320737472617465677920413c2f626f64793e3c2f68746d6c3e0a warcgz/data.bin=766572626174696d20677a6970206f637465742d73747265616d20626f647920666f722074686520574152432069735f777269746520706174680a"
|
||||
export WARC_VALIDATE_VERBATIM=1
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --warc-validate \
|
||||
--found 'warcgz/index.html' --found 'warcgz/page.html' --found 'warcgz/data.bin' \
|
||||
httrack 'BASEURL/warcgz/index.html' --warc-file warc-out
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A --wacz crawl packages the WARC archive, its CDXJ index and a generated
|
||||
# pages.jsonl into a single WACZ file at crawl end. The stdlib validator is the
|
||||
# real gate: STORE-mode entries, the fixed layout, recomputed sha256 digests and
|
||||
# the datapackage-digest chain (plus py-wacz/pywb when importable). The crawl
|
||||
# skips cleanly on a build without OpenSSL (no conformant SHA-256 -> no package).
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --wacz-validate \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --wacz
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# #133: the save-path ceiling used to be the Windows MAX_PATH (236 usable) on
|
||||
# every platform, needlessly hashing long names on Linux/macOS/Android. Off
|
||||
# Windows a name well under the platform PATH_MAX must now be kept whole.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
# ~300 chars: over the old 236 ceiling, well under any POSIX PATH_MAX. The
|
||||
# distinctive 90-char last segment lets us tell "kept whole" from "hashed".
|
||||
seg="$(printf 'Z%.0s' {1..90})"
|
||||
deep="/a/$(printf 'b%.0s' {1..100})/c/$(printf 'd%.0s' {1..100})"
|
||||
out="$("$httrack_bin" -O /dev/null -#test=savename "$deep/$seg.html" text/html |
|
||||
sed -n 's/^savename: //p')"
|
||||
|
||||
case "$(uname -s 2>/dev/null)" in
|
||||
MINGW* | MSYS* | CYGWIN*)
|
||||
# Windows still caps at MAX_PATH: the distinctive tail is cut
|
||||
case "$out" in
|
||||
*"$seg"*)
|
||||
echo "FAIL: Windows should have shortened the long name: '$out'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
# POSIX: the full name survives, no hashing
|
||||
case "$out" in
|
||||
*"$seg"*) ;;
|
||||
*)
|
||||
echo "FAIL: long POSIX name was truncated: '$out'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
test "${#out}" -gt 236 ||
|
||||
{
|
||||
echo "FAIL: POSIX name not longer than the old ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# byte-safety: a multibyte name whose codepoint count is under the ceiling
|
||||
# but whose bytes plus a long -O dir overflow the save buffer must be cut to
|
||||
# fit, not abort the crawl (SIGABRT before the byte-cut guard).
|
||||
bigdir="/x/$(printf 'D%.0s' $(seq 1 1015))" # ~1020 bytes
|
||||
cjk="/$(printf '\xe4\xb8\xad%.0s' $(seq 1 339))" # 1018 bytes, 339 codepoints
|
||||
if ! mb="$("$httrack_bin" -O "$bigdir" -#test=savename "$cjk" text/html 2>/dev/null)"; then
|
||||
echo "FAIL: aborted on a multibyte over-buffer name"
|
||||
exit 1
|
||||
fi
|
||||
mb="${mb#savename: }"
|
||||
if [ -z "$mb" ] || [ "${#mb}" -ge 2048 ]; then
|
||||
echo "FAIL: multibyte name not bounded (len=${#mb})"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "longpath-posix OK"
|
||||
@@ -3,7 +3,7 @@
|
||||
# silently drop it from the dist tarball and break "make distcheck".
|
||||
EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
proxy-https-server.py socks5-server.py proxy-connect-server.py \
|
||||
proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \
|
||||
proxytestlib.py tls-stall-server.py warc-validate.py \
|
||||
local-crawl.sh local-server.py testlib.sh server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
@@ -64,10 +64,6 @@ TESTS = \
|
||||
01_engine-expandhome.test \
|
||||
01_engine-fsize.test \
|
||||
01_engine-redirect.test \
|
||||
01_engine-longpath-io.test \
|
||||
01_engine-mirror-io.test \
|
||||
01_engine-direnum.test \
|
||||
01_engine-cookieimport.test \
|
||||
01_engine-relative.test \
|
||||
01_engine-robots.test \
|
||||
01_engine-savename.test \
|
||||
@@ -82,12 +78,9 @@ TESTS = \
|
||||
01_engine-unescape-bounds.test \
|
||||
01_engine-useragent.test \
|
||||
01_engine-version-macros.test \
|
||||
01_engine-warc-surt.test \
|
||||
01_engine-xfread.test \
|
||||
01_zlib-acceptencoding.test \
|
||||
01_zlib-warc.test \
|
||||
01_zlib-warc-cdx.test \
|
||||
01_zlib-warc-wacz.test \
|
||||
01_zlib-contentcodings.test \
|
||||
01_zlib-cache.test \
|
||||
01_zlib-cache-corrupt.test \
|
||||
@@ -164,9 +157,6 @@ TESTS = \
|
||||
69_local-intl-logdir.test \
|
||||
71_local-crange-repaircache.test \
|
||||
72_watchdog-crawl.test \
|
||||
73_local-warc.test \
|
||||
74_local-warc-wacz.test \
|
||||
74_local-warc-verbatim.test \
|
||||
75_engine-longpath-posix.test
|
||||
73_local-warc.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -48,7 +48,6 @@ key="${testdir}/server.key"
|
||||
tls=
|
||||
verbose=
|
||||
warc_validate=
|
||||
wacz_validate=
|
||||
html_subdir=
|
||||
outdir_intl=
|
||||
rerun=
|
||||
@@ -118,8 +117,6 @@ while test "$pos" -lt "$nargs"; do
|
||||
--rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
|
||||
# validate the produced .warc.gz (see the validation block near the end)
|
||||
--warc-validate) warc_validate=1 ;;
|
||||
# validate the produced .wacz package (stdlib, plus py-wacz/pywb if present)
|
||||
--wacz-validate) wacz_validate=1 ;;
|
||||
--no-purge)
|
||||
nopurge=1
|
||||
audit+=("--no-purge")
|
||||
@@ -375,13 +372,9 @@ if test -n "$warc_validate"; then
|
||||
fresh="${tmpdir}/warc-pass1.gz"
|
||||
test -f "$fresh" || fresh="$warc"
|
||||
declare -a bodyargs=()
|
||||
# WARC_VALIDATE_BODY holds one or more whitespace-separated SUB=HEX specs.
|
||||
for spec in ${WARC_VALIDATE_BODY:-}; do
|
||||
bodyargs+=(--expect-body-hex "$spec")
|
||||
done
|
||||
# compressed asset: assert the stored (verbatim) body inflates to the served
|
||||
# body and keeps Content-Encoding, instead of expecting a decoded body.
|
||||
test -n "${WARC_VALIDATE_VERBATIM:-}" && bodyargs+=(--verbatim)
|
||||
if test -n "${WARC_VALIDATE_BODY:-}"; then
|
||||
bodyargs=(--expect-body-hex "$WARC_VALIDATE_BODY")
|
||||
fi
|
||||
info "validating fresh WARC (response bodies)"
|
||||
"$python" "$validator" "$(nativepath "$fresh")" "${bodyargs[@]}" >&2 ||
|
||||
die "fresh WARC validation failed"
|
||||
@@ -405,23 +398,6 @@ if test -n "$warc_validate"; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- optional WACZ validation (--wacz) --------------------------------------
|
||||
if test -n "$wacz_validate"; then
|
||||
wacz=$(find "$mirrorroot" -maxdepth 2 -name '*.wacz' 2>/dev/null | sort | tail -n1)
|
||||
if test -z "$wacz"; then
|
||||
# No package: only acceptable when the build lacks OpenSSL (SHA-256).
|
||||
if grep -aqi "WACZ requires an OpenSSL" "${logroot}/hts-log.txt"; then
|
||||
info "no .wacz produced (build without OpenSSL); skipping"
|
||||
exit 77
|
||||
fi
|
||||
die "no .wacz file produced under $mirrorroot"
|
||||
fi
|
||||
validator=$(nativepath "${testdir}/wacz-validate.py")
|
||||
info "validating WACZ package"
|
||||
"$python" "$validator" "$(nativepath "$wacz")" >&2 || die "WACZ validation failed"
|
||||
result "OK"
|
||||
fi
|
||||
|
||||
# No crawl, even a cancelled one, may leave engine temporaries: .delayed (#107,
|
||||
# #483), or the .z/.u content-coding temps (#557).
|
||||
info "checking for leftover engine temporaries"
|
||||
|
||||
@@ -635,34 +635,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
extra_headers=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
|
||||
# A gzip-coded HTML page whose decoded body is known, for the verbatim-WARC
|
||||
# differential (stored compressed bytes must inflate to this).
|
||||
WARCGZ_BODY = b"<html><body>verbatim gzip page for WARC strategy A</body></html>\n"
|
||||
|
||||
# A NON-html gzip-coded asset: HTTrack streams it straight to disk, so the
|
||||
# verbatim spool adoption runs on the is_write (direct-to-disk) branch of
|
||||
# back_finalize, not the in-memory branch route_warcgz_page exercises.
|
||||
WARCGZ_BIN_BODY = b"verbatim gzip octet-stream body for the WARC is_write path\n"
|
||||
|
||||
def route_warcgz_index(self):
|
||||
self.send_html(
|
||||
'\t<a href="page.html">page</a>\n' '\t<a href="data.bin">data</a>\n'
|
||||
)
|
||||
|
||||
def route_warcgz_page(self):
|
||||
self.send_raw(
|
||||
gzip.compress(self.WARCGZ_BODY),
|
||||
"text/html",
|
||||
extra_headers=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
|
||||
def route_warcgz_data(self):
|
||||
self.send_raw(
|
||||
gzip.compress(self.WARCGZ_BIN_BODY),
|
||||
"application/octet-stream",
|
||||
extra_headers=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
|
||||
# --- content codings ---------------------------------------------------
|
||||
# Canned br/zstd bodies (no brotli/zstd module in the stdlib): both decode
|
||||
# to CODEC_BODY. Regenerate with the brotli/zstd CLIs over that string.
|
||||
@@ -1570,9 +1542,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/gated/index.php": route_gated_index,
|
||||
"/gated/secret.php": route_gated_secret,
|
||||
"/robots.txt": route_robots,
|
||||
"/warcgz/index.html": route_warcgz_index,
|
||||
"/warcgz/page.html": route_warcgz_page,
|
||||
"/warcgz/data.bin": route_warcgz_data,
|
||||
"/codec/index.html": route_codec_index,
|
||||
"/codec/br.html": route_codec_br,
|
||||
"/codec/zstd.html": route_codec_zstd,
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Validate a WACZ package with the stdlib only (no py-wacz needed): every entry
|
||||
# is ZIP STORE, the fixed layout is present, each datapackage resource hash and
|
||||
# size recomputes, the digest chains datapackage.json, and pages.jsonl carries
|
||||
# the json-pages-1.0 header. If py-wacz (`wacz validate`) is importable it runs
|
||||
# too; both gates must pass. Exit 0 = valid, nonzero = invalid.
|
||||
import sys
|
||||
import json
|
||||
import zipfile
|
||||
import hashlib
|
||||
|
||||
|
||||
def fail(msg):
|
||||
print("wacz-validate: FAIL: %s" % msg, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
fail("usage: wacz-validate.py FILE.wacz")
|
||||
path = sys.argv[1]
|
||||
z = zipfile.ZipFile(path)
|
||||
names = z.namelist()
|
||||
|
||||
for info in z.infolist():
|
||||
if info.compress_type != zipfile.ZIP_STORED:
|
||||
fail("%s is not STORE mode (%d)" % (info.filename, info.compress_type))
|
||||
|
||||
need_arc = any(
|
||||
n.startswith("archive/") and n.endswith((".warc.gz", ".warc")) for n in names
|
||||
)
|
||||
for req, ok in (
|
||||
("archive/*.warc.gz", need_arc),
|
||||
("indexes/index.cdx", "indexes/index.cdx" in names),
|
||||
("pages/pages.jsonl", "pages/pages.jsonl" in names),
|
||||
("datapackage.json", "datapackage.json" in names),
|
||||
("datapackage-digest.json", "datapackage-digest.json" in names),
|
||||
):
|
||||
if not ok:
|
||||
fail("missing %s (entries: %s)" % (req, names))
|
||||
|
||||
lines = [ln for ln in z.read("pages/pages.jsonl").split(b"\n") if ln.strip()]
|
||||
if not lines or json.loads(lines[0]).get("format") != "json-pages-1.0":
|
||||
fail("pages.jsonl header is not json-pages-1.0: %r" % lines[:1])
|
||||
body = [json.loads(ln) for ln in lines[1:]]
|
||||
if not body:
|
||||
fail("pages.jsonl has no page rows after the header")
|
||||
for row in body:
|
||||
if "url" not in row or "ts" not in row:
|
||||
fail("pages.jsonl row missing url/ts: %r" % row)
|
||||
|
||||
dp = json.loads(z.read("datapackage.json"))
|
||||
if dp.get("profile") != "data-package":
|
||||
fail("profile != data-package: %r" % dp.get("profile"))
|
||||
if dp.get("wacz_version") != "1.1.1":
|
||||
fail("wacz_version != 1.1.1: %r" % dp.get("wacz_version"))
|
||||
resources = dp.get("resources", [])
|
||||
if not resources:
|
||||
fail("datapackage has no resources")
|
||||
for r in resources:
|
||||
data = z.read(r["path"])
|
||||
h = "sha256:" + hashlib.sha256(data).hexdigest()
|
||||
if h != r["hash"]:
|
||||
fail("%s hash %s != %s" % (r["path"], h, r["hash"]))
|
||||
if len(data) != r["bytes"]:
|
||||
fail("%s bytes %d != %d" % (r["path"], len(data), r["bytes"]))
|
||||
|
||||
dig = json.loads(z.read("datapackage-digest.json"))
|
||||
want = "sha256:" + hashlib.sha256(z.read("datapackage.json")).hexdigest()
|
||||
if dig.get("path") != "datapackage.json" or dig.get("hash") != want:
|
||||
fail("digest chain broken: %r" % dig)
|
||||
|
||||
print(
|
||||
"wacz-validate: OK (%d entries, %d resources, stdlib)"
|
||||
% (len(names), len(resources))
|
||||
)
|
||||
|
||||
# Optional stricter gate when py-wacz is present.
|
||||
try:
|
||||
import wacz # noqa: F401
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
from wacz.main import main as wacz_main # type: ignore
|
||||
except Exception:
|
||||
return
|
||||
print("wacz-validate: running py-wacz validate")
|
||||
rc = wacz_main(["validate", "-f", path])
|
||||
if rc not in (0, None):
|
||||
fail("py-wacz validate returned %r" % rc)
|
||||
print("wacz-validate: py-wacz OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -11,11 +11,6 @@
|
||||
# WARC-Payload-Digest matching sha1(body) when present
|
||||
# --no-response-for SUB the asset containing SUB must be a revisit: no
|
||||
# response may target it, and a revisit must
|
||||
# --verbatim compressed asset: --expect-body-hex instead keeps
|
||||
# Content-Encoding, checks the HTTP Content-Length is
|
||||
# the stored (compressed) length, asserts the stored
|
||||
# body inflates to HEX (the served plaintext), and
|
||||
# requires the payload digest when the file emits any
|
||||
import base64
|
||||
import hashlib
|
||||
import sys
|
||||
@@ -66,53 +61,14 @@ def check_body(rec, http_hdr, body, sub, want):
|
||||
sys.exit("WARC-Payload-Digest mismatch for %s" % sub)
|
||||
|
||||
|
||||
def check_body_verbatim(rec, http_hdr, body, sub, want, digests_emitted):
|
||||
"""Verbatim: the stored body is the coded octets, Content-Encoding is kept,
|
||||
the HTTP Content-Length equals the stored (compressed) length, inflating the
|
||||
body yields the served plaintext, and the payload digest is over the coded
|
||||
body. The differential: inflate(stored) == the body the server compressed."""
|
||||
if b"Content-Encoding" not in http_hdr:
|
||||
sys.exit("verbatim record for %s dropped Content-Encoding" % sub)
|
||||
if b"Transfer-Encoding" in http_hdr:
|
||||
sys.exit("verbatim record for %s kept Transfer-Encoding" % sub)
|
||||
hcl = field(http_hdr, b"Content-Length")
|
||||
if hcl is None or int(hcl) != len(body):
|
||||
sys.exit("verbatim record for %s: HTTP Content-Length != stored body" % sub)
|
||||
try:
|
||||
decoded = zlib.decompress(body, zlib.MAX_WBITS | 16)
|
||||
except Exception as exc:
|
||||
sys.exit("verbatim record for %s: body did not inflate: %s" % (sub, exc))
|
||||
if decoded != want:
|
||||
sys.exit(
|
||||
"verbatim decoded mismatch for %s: got %d bytes, expected %d"
|
||||
% (sub, len(decoded), len(want))
|
||||
)
|
||||
pd = field(rec[: rec.find(b"\r\n\r\n")], b"WARC-Payload-Digest")
|
||||
# Catch a regression that drops the digest on the verbatim path, but only
|
||||
# when this file emits digests at all (an OpenSSL build; none otherwise).
|
||||
if digests_emitted and pd is None:
|
||||
sys.exit("verbatim record for %s: missing WARC-Payload-Digest" % sub)
|
||||
if pd is not None and pd.startswith(b"sha1:"):
|
||||
want_b32 = base64.b32encode(hashlib.sha1(body).digest()).decode("ascii")
|
||||
if pd[5:].decode("ascii") != want_b32:
|
||||
sys.exit("WARC-Payload-Digest (compressed) mismatch for %s" % sub)
|
||||
|
||||
|
||||
def main():
|
||||
argv = sys.argv[1:]
|
||||
expect_revisit = "--expect-revisit" in argv
|
||||
verbatim = "--verbatim" in argv
|
||||
body_specs = [s.split("=", 1) for s in opt_values(argv, "--expect-body-hex")]
|
||||
no_resp = opt_values(argv, "--no-response-for")
|
||||
path = [a for a in argv if not a.startswith("--") and "=" not in a][0]
|
||||
data = open(path, "rb").read()
|
||||
|
||||
# digests are emitted only on an OpenSSL build; detect it once so --verbatim
|
||||
# can require the payload digest exactly when the file carries any.
|
||||
digests_emitted = any(
|
||||
b"WARC-Payload-Digest" in r[: r.find(b"\r\n\r\n")] for r in records(data)
|
||||
)
|
||||
|
||||
total = revisits = responses = infos = 0
|
||||
body_hits = {sub: False for sub, _ in body_specs}
|
||||
revisit_hits = {sub: False for sub in no_resp}
|
||||
@@ -150,13 +106,7 @@ def main():
|
||||
http_hdr, body = block[:bsep], block[bsep + 4 :]
|
||||
for sub, hexval in body_specs:
|
||||
if sub.encode() in uri:
|
||||
want = bytes.fromhex(hexval)
|
||||
if verbatim:
|
||||
check_body_verbatim(
|
||||
rec, http_hdr, body, sub, want, digests_emitted
|
||||
)
|
||||
else:
|
||||
check_body(rec, http_hdr, body, sub, want)
|
||||
check_body(rec, http_hdr, body, sub, bytes.fromhex(hexval))
|
||||
body_hits[sub] = True
|
||||
elif wtype == b"revisit":
|
||||
revisits += 1
|
||||
|
||||
Reference in New Issue
Block a user