mirror of
https://github.com/xroche/httrack.git
synced 2026-07-23 09:09:05 +03:00
Compare commits
2 Commits
master
...
feat/webht
| Author | SHA1 | Date | |
|---|---|---|---|
|
4ae6124257 | ||
|
|
76dcf61d20 |
@@ -90,34 +90,35 @@ offline browser : copy websites to a local directory</p>
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
<b>-%x, --disable-passwords</b> ] [ <b>-%q,
|
||||
--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
|
||||
<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
|
||||
] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
|
||||
] [ <b>-bN, --cookies[=N]</b> ] [ <b>-%K, --cookies-file</b>
|
||||
] [ <b>-%Y, --why</b> ] [ <b>-u, --check-type[=N]</b> ] [
|
||||
<b>-j, --parse-java[=N]</b> ] [ <b>-sN, --robots[=N]</b> ] [
|
||||
<b>-%h, --http-10</b> ] [ <b>-%k, --keep-alive</b> ] [
|
||||
<b>-%z, --disable-compression</b> ] [ <b>-%B, --tolerant</b>
|
||||
] [ <b>-%s, --updatehack</b> ] [ <b>-%u, --urlhack</b> ] [
|
||||
] [ <b>-%r, --warc</b> ] [ <b>-LN, --long-names[=N]</b> ] [
|
||||
<b>-KN, --keep-links[=N]</b> ] [ <b>-x,
|
||||
--replace-external</b> ] [ <b>-%x, --disable-passwords</b> ]
|
||||
[ <b>-%q, --include-query-string</b> ] [ <b>-%g,
|
||||
--strip-query</b> ] [ <b>-o, --generate-errors</b> ] [
|
||||
<b>-X, --purge-old[=N]</b> ] [ <b>-%p, --preserve</b> ] [
|
||||
<b>-%T, --utf8-conversion</b> ] [ <b>-bN, --cookies[=N]</b>
|
||||
] [ <b>-%K, --cookies-file</b> ] [ <b>-%Y, --why</b> ] [
|
||||
<b>-u, --check-type[=N]</b> ] [ <b>-j, --parse-java[=N]</b>
|
||||
] [ <b>-sN, --robots[=N]</b> ] [ <b>-%h, --http-10</b> ] [
|
||||
<b>-%k, --keep-alive</b> ] [ <b>-%z,
|
||||
--disable-compression</b> ] [ <b>-%B, --tolerant</b> ] [
|
||||
<b>-%s, --updatehack</b> ] [ <b>-%u, --urlhack</b> ] [
|
||||
<b>-%A, --assume</b> ] [ <b>-@iN, --protocol[=N]</b> ] [
|
||||
<b>-%w, --disable-module</b> ] [ <b>-F, --user-agent</b> ] [
|
||||
<b>-%R, --referer</b> ] [ <b>-%E, --from</b> ] [ <b>-%F,
|
||||
--footer</b> ] [ <b>-%l, --language</b> ] [ <b>-%a,
|
||||
--accept</b> ] [ <b>-%X, --headers</b> ] [ <b>-C,
|
||||
--cache[=N]</b> ] [ <b>-k, --store-all-in-cache</b> ] [
|
||||
<b>-%r, --warc</b> ] [ <b>-%n, --do-not-recatch</b> ] [
|
||||
<b>-%v, --display</b> ] [ <b>-Q, --do-not-log</b> ] [ <b>-q,
|
||||
--quiet</b> ] [ <b>-z, --extra-log</b> ] [ <b>-Z,
|
||||
--debug-log</b> ] [ <b>-v, --verbose</b> ] [ <b>-f,
|
||||
--file-log</b> ] [ <b>-f2, --single-log</b> ] [ <b>-I,
|
||||
--index</b> ] [ <b>-%i, --build-top-index</b> ] [ <b>-%I,
|
||||
--search-index</b> ] [ <b>-pN, --priority[=N]</b> ] [ <b>-S,
|
||||
--stay-on-same-dir</b> ] [ <b>-D, --can-go-down</b> ] [
|
||||
<b>-U, --can-go-up</b> ] [ <b>-B, --can-go-up-and-down</b> ]
|
||||
[ <b>-a, --stay-on-same-address</b> ] [ <b>-d,
|
||||
<b>-%n, --do-not-recatch</b> ] [ <b>-%v, --display</b> ] [
|
||||
<b>-Q, --do-not-log</b> ] [ <b>-q, --quiet</b> ] [ <b>-z,
|
||||
--extra-log</b> ] [ <b>-Z, --debug-log</b> ] [ <b>-v,
|
||||
--verbose</b> ] [ <b>-f, --file-log</b> ] [ <b>-f2,
|
||||
--single-log</b> ] [ <b>-I, --index</b> ] [ <b>-%i,
|
||||
--build-top-index</b> ] [ <b>-%I, --search-index</b> ] [
|
||||
<b>-pN, --priority[=N]</b> ] [ <b>-S, --stay-on-same-dir</b>
|
||||
] [ <b>-D, --can-go-down</b> ] [ <b>-U, --can-go-up</b> ] [
|
||||
<b>-B, --can-go-up-and-down</b> ] [ <b>-a,
|
||||
--stay-on-same-address</b> ] [ <b>-d,
|
||||
--stay-on-same-domain</b> ] [ <b>-l, --stay-on-same-tld</b>
|
||||
] [ <b>-e, --go-everywhere</b> ] [ <b>-%H,
|
||||
--debug-headers</b> ] [ <b>-%!,
|
||||
@@ -648,6 +649,18 @@ don’t wait) (--cached-delayed-type-check)</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%r</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
|
||||
sets the output name (--warc)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%t</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
@@ -1129,20 +1142,6 @@ update before) (--cache[=N])</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%r</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
|
||||
sets the output name, --warc-max-size N rotates segments
|
||||
past N bytes, --warc-cdx also writes a sorted CDXJ index,
|
||||
--wacz packages it all as a WACZ file (--warc)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%n</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "23 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "22 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -40,6 +40,7 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-%r, \-\-warc\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
[ \fB\-KN, \-\-keep\-links[=N]\fR ]
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
@@ -74,7 +75,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%X, \-\-headers\fR ]
|
||||
[ \fB\-C, \-\-cache[=N]\fR ]
|
||||
[ \fB\-k, \-\-store\-all\-in\-cache\fR ]
|
||||
[ \fB\-%r, \-\-warc\fR ]
|
||||
[ \fB\-%n, \-\-do\-not\-recatch\fR ]
|
||||
[ \fB\-%v, \-\-display\fR ]
|
||||
[ \fB\-Q, \-\-do\-not\-log\fR ]
|
||||
@@ -198,6 +198,8 @@ delayed type check, don't make any link test but wait for files download to star
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%r
|
||||
write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name (\-\-warc)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
@@ -277,8 +279,6 @@ additional HTTP header line (\-%X "X\-Magic: 42" (\-\-headers <param>)
|
||||
create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before) (\-\-cache[=N])
|
||||
.IP \-k
|
||||
store all files in cache (not useful if files on disk) (\-\-store\-all\-in\-cache)
|
||||
.IP \-%r
|
||||
write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name, \-\-warc\-max\-size N rotates segments past N bytes, \-\-warc\-cdx also writes a sorted CDXJ index, \-\-wacz packages it all as a WACZ file (\-\-warc)
|
||||
.IP \-%n
|
||||
do not re\-download locally erased files (\-\-do\-not\-recatch)
|
||||
.IP \-%v
|
||||
|
||||
@@ -116,13 +116,6 @@ const char *hts_optalias[][4] = {
|
||||
"load extra cookies from a Netscape cookies.txt"},
|
||||
{"warc", "-%r", "single", "write an ISO-28500 WARC/1.1 archive of the crawl"},
|
||||
{"warc-file", "-%rf", "param1", "write a WARC archive to the given base name"},
|
||||
{"warc-max-size", "-%rs", "param1",
|
||||
"rotate the WARC archive once a segment passes N bytes (0: single file)"},
|
||||
{"warc-cdx", "-%rc", "single",
|
||||
"write a sorted CDXJ index next to the WARC archive"},
|
||||
{"warc-cdxj", "-%rc", "single", ""},
|
||||
{"wacz", "-%rz", "single",
|
||||
"package the WARC archive, CDXJ index and pages as a WACZ file"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
|
||||
@@ -750,20 +750,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
fexist_utf8(back[p].url_sav))
|
||||
filenote(&opt->state.strc, back[p].url_sav, NULL);
|
||||
}
|
||||
/* Keep the compressed spool so the WARC record stores the body
|
||||
verbatim (Content-Encoding preserved) instead of unlinking it.
|
||||
*/
|
||||
if (StringNotEmpty(opt->warc_file)) {
|
||||
warc_adopt_rawspool(&back[p].r, back[p].tmpfile);
|
||||
if (back[p].r.warc_rawpath != NULL)
|
||||
back[p].tmpfile =
|
||||
NULL; /* adopted: freed via warc_free_request */
|
||||
}
|
||||
/* ensure that no remaining temporary file exists */
|
||||
if (back[p].tmpfile != NULL) {
|
||||
unlink(back[p].tmpfile);
|
||||
back[p].tmpfile = NULL;
|
||||
}
|
||||
unlink(back[p].tmpfile);
|
||||
back[p].tmpfile = NULL;
|
||||
}
|
||||
// stats
|
||||
HTS_STAT.total_packed += back[p].compressed_size;
|
||||
@@ -1073,9 +1062,6 @@ void back_copy_static(const lien_back * src, lien_back * dst) {
|
||||
dst->r.headers = NULL;
|
||||
dst->r.warc_reqhdr = NULL;
|
||||
dst->r.warc_resphdr = NULL;
|
||||
dst->r.warc_rawpath =
|
||||
NULL; /* the spool stays owned by src (no double-unlink) */
|
||||
dst->r.warc_truncated = 0;
|
||||
dst->r.out = NULL;
|
||||
dst->r.location = dst->location_buffer;
|
||||
dst->r.fp = NULL;
|
||||
@@ -1141,8 +1127,6 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
|
||||
(*dst)->r.out = NULL;
|
||||
(*dst)->r.warc_reqhdr = NULL;
|
||||
(*dst)->r.warc_resphdr = NULL;
|
||||
(*dst)->r.warc_rawpath = NULL;
|
||||
(*dst)->r.warc_truncated = 0;
|
||||
(*dst)->r.location = (*dst)->location_buffer;
|
||||
(*dst)->r.fp = NULL;
|
||||
(*dst)->r.soc = INVALID_SOCKET;
|
||||
@@ -2535,19 +2519,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
|
||||
for (i = 0; i < (unsigned int) back_max; i++) {
|
||||
if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
|
||||
/* A cap-truncated body is deliberate, not broken: archive what arrived
|
||||
with WARC-Truncated before the abort overwrites the slot's real 2xx
|
||||
status. HTTrack still treats the slot as incomplete afterwards. */
|
||||
if (StringNotEmpty(opt->warc_file) && back[i].r.statuscode > 0 &&
|
||||
back[i].r.warc_resphdr != NULL && back[i].r.size > 0 &&
|
||||
!(back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))) {
|
||||
if (back[i].r.is_write && back[i].r.out != NULL)
|
||||
fflush(back[i].r.out);
|
||||
back[i].r.warc_truncated = (limit == HTS_MIRROR_LIMIT_SIZE)
|
||||
? WARC_TRUNC_LENGTH
|
||||
: WARC_TRUNC_TIME;
|
||||
warc_write_backtransaction(opt, &back[i]);
|
||||
}
|
||||
if (back[i].r.soc != INVALID_SOCKET) {
|
||||
deletehttp(&back[i].r);
|
||||
}
|
||||
|
||||
@@ -3632,9 +3632,6 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
|
||||
if (StringNotEmpty(from->warc_file))
|
||||
StringCopyS(to->warc_file, from->warc_file);
|
||||
to->warc_max_size = from->warc_max_size;
|
||||
to->warc_cdx = from->warc_cdx;
|
||||
to->warc_wacz = from->warc_wacz;
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
|
||||
@@ -1806,34 +1806,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
return -1;
|
||||
}
|
||||
StringCopy(opt->warc_file, argv[na]);
|
||||
} else if (*(com + 1) == 's') { // --warc-max-size N: rotation
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option warc-max-size needs a blank space and a size");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
{ // reject non-numeric/negative/overflow; keep default 0
|
||||
// (single file)
|
||||
char *end;
|
||||
LLint v;
|
||||
errno = 0;
|
||||
v = strtoll(argv[na], &end, 10);
|
||||
if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
|
||||
errno != ERANGE)
|
||||
opt->warc_max_size = v;
|
||||
}
|
||||
} else if (*(com + 1) == 'c') { // --warc-cdx: sorted CDXJ index
|
||||
com++;
|
||||
opt->warc_cdx = 1;
|
||||
} else if (*(com + 1) == 'z') { // --wacz: WACZ package
|
||||
com++;
|
||||
opt->warc_wacz = 1;
|
||||
opt->warc_cdx = 1; // WACZ embeds the CDXJ index
|
||||
if (!StringNotEmpty(opt->warc_file))
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
} else { // --warc: auto-named archive under the output dir
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
|
||||
@@ -524,6 +524,8 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %r write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
|
||||
"output name");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
@@ -593,10 +595,6 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" C create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)");
|
||||
infomsg(" k store all files in cache (not useful if files on disk)");
|
||||
infomsg(" %r write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
|
||||
"output name, --warc-max-size N rotates segments past N bytes, "
|
||||
"--warc-cdx also writes a sorted CDXJ index, --wacz packages it all "
|
||||
"as a WACZ file");
|
||||
infomsg(" %n do not re-download locally erased files");
|
||||
infomsg
|
||||
(" %v display on screen filenames downloaded (in realtime) - * %v1 short version - %v2 full animation");
|
||||
|
||||
@@ -6019,7 +6019,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->strip_query, "");
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->warc_file, "");
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
|
||||
11
src/htsopt.h
11
src/htsopt.h
@@ -541,12 +541,6 @@ struct httrackp {
|
||||
and exit without crawling */
|
||||
String warc_file; /**< WARC output: WARC_AUTONAME for --warc, or the
|
||||
--warc-file basename (appended at the tail: ABI) */
|
||||
LLint warc_max_size; /**< --warc-max-size: rotate the archive past this many
|
||||
bytes (<=0: single file). Tail: ABI */
|
||||
hts_boolean warc_cdx; /**< --warc-cdx: write a sorted CDXJ index next to the
|
||||
archive. Tail: ABI */
|
||||
hts_boolean warc_wacz; /**< --wacz: package archive+index+pages as a WACZ zip
|
||||
(implies --warc + --warc-cdx). Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
@@ -673,11 +667,6 @@ struct htsblk {
|
||||
char *warc_reqhdr; /**< stashed raw request header block for WARC (or NULL) */
|
||||
char *
|
||||
warc_resphdr; /**< stashed raw response header block for WARC (or NULL) */
|
||||
int warc_truncated; /**< WARC-Truncated reason for a cap-truncated body
|
||||
(WARC_TRUNC_*, 0=none). Tail: ABI */
|
||||
char *warc_rawpath; /**< verbatim WARC: spooled compressed body path, or NULL
|
||||
(owns the file; unlinked on free). Tail: ABI */
|
||||
LLint warc_rawsize; /**< byte length of warc_rawpath. Tail: ABI */
|
||||
/*char digest[32+2]; // md5 digest generated by the engine ("" if none) */
|
||||
};
|
||||
|
||||
|
||||
@@ -64,9 +64,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#if HTS_USEZSTD
|
||||
#include <zstd.h>
|
||||
#endif
|
||||
#if HTS_USEOPENSSL
|
||||
#include <openssl/evp.h>
|
||||
#endif
|
||||
#include "coucal/coucal.h"
|
||||
|
||||
#include <ctype.h>
|
||||
@@ -3357,8 +3354,7 @@ static unsigned char *warc_next_member(const unsigned char **in,
|
||||
/* Feed a synthetic transaction and validate the resulting .warc.gz against the
|
||||
WARC/1.1 spec: each record a self-standing gzip member starting WARC/1.,
|
||||
Content-Length == block length, the \r\n\r\n trailer intact, the response
|
||||
body round-trips, and the hop-by-hop Transfer-Encoding is dropped (a real
|
||||
Content-Encoding is kept verbatim; see warc-verbatim). */
|
||||
body round-trips, and the encoding headers are stripped (strategy B). */
|
||||
static int st_warc(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
@@ -3378,22 +3374,21 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
|
||||
/* 200 HTML, plaintext body: bogus Content-Length rewritten, hop-by-hop
|
||||
Transfer-Encoding dropped. The whitespace before its ':' exercises
|
||||
header_is tolerating "Name : value". */
|
||||
/* 200 HTML: bogus Content-Length + gzip/chunked encodings must be stripped.
|
||||
*/
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/a.html", "127.0.0.1",
|
||||
"GET /a.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n"
|
||||
"Transfer-Encoding : chunked\r\nContent-Length: 999\r\n\r\n",
|
||||
a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
|
||||
"gzip\r\nTransfer-Encoding: chunked\r\nContent-Length: 999\r\n\r\n",
|
||||
a_body, sizeof(a_body) - 1, NULL, 200, 0);
|
||||
|
||||
/* 302 redirect: header-only, no body. */
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/r", "127.0.0.1",
|
||||
"GET /r HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 302 Found\r\nLocation: http://test.local/a.html\r\n\r\n", NULL,
|
||||
0, NULL, 302, 0, 0);
|
||||
0, NULL, 302, 0);
|
||||
|
||||
/* 200 binary, chunked coding on the wire (already de-chunked here). */
|
||||
warc_write_transaction(
|
||||
@@ -3401,7 +3396,7 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
|
||||
"GET /b.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n"
|
||||
"Transfer-Encoding: chunked\r\n\r\n",
|
||||
"\x00\x01\x02\x03\x04", 5, NULL, 200, 0, 0);
|
||||
"\x00\x01\x02\x03\x04", 5, NULL, 200, 0);
|
||||
|
||||
/* 200 with a body shorter than the declared Content-Length (rewritten). */
|
||||
warc_write_transaction(
|
||||
@@ -3409,20 +3404,20 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
|
||||
"GET /trunc HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: "
|
||||
"100\r\n\r\n",
|
||||
"short", 5, NULL, 200, 0, 0);
|
||||
"short", 5, NULL, 200, 0);
|
||||
|
||||
/* Same payload as a.html at a new URL: identical-payload-digest revisit
|
||||
(OpenSSL builds only; a plain build writes a second full response). */
|
||||
warc_write_transaction(w, "http://test.local/a2.html", "127.0.0.1",
|
||||
"GET /a2.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
|
||||
a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
|
||||
a_body, sizeof(a_body) - 1, NULL, 200, 0);
|
||||
|
||||
/* 304 revisit with an EMPTY response-header block: the block is just the
|
||||
2-byte separator, so declared Content-Length must be exactly 2 (F3). */
|
||||
warc_write_transaction(w, "http://test.local/nm", "127.0.0.1",
|
||||
"GET /nm HTTP/1.1\r\nHost: test.local\r\n\r\n", "",
|
||||
NULL, 0, NULL, 304, 1, 0);
|
||||
NULL, 0, NULL, 304, 1);
|
||||
|
||||
warc_close(w);
|
||||
|
||||
@@ -3490,8 +3485,7 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
|
||||
hdr_len, 37) != NULL &&
|
||||
warc_memstr((char *) rec, "WARC-Type: revisit", hdr_len, 18) != NULL)
|
||||
nm_cl_ok = (block_len == 2);
|
||||
/* a.html response body round-trips; no Content-Encoding (plaintext) and the
|
||||
whitespaced Transfer-Encoding was dropped (header_is robustness). */
|
||||
/* a.html response body must round-trip and carry no encoding headers */
|
||||
if (warc_memstr((char *) rec, "WARC-Target-URI: http://test.local/a.html",
|
||||
hdr_len, 41) != NULL &&
|
||||
warc_memstr((char *) rec, "msgtype=response", hdr_len, 16) != NULL) {
|
||||
@@ -3540,770 +3534,6 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Parse a record's header/block split; sets *hdr_len and *block_len, returns 0
|
||||
when Content-Length matches the actual block bytes, -1 otherwise. */
|
||||
static int warc_rec_split(const unsigned char *rec, size_t rlen,
|
||||
size_t *hdr_len, long long *block_len) {
|
||||
const char *sep = warc_memstr((const char *) rec, "\r\n\r\n", rlen, 4);
|
||||
const char *cl;
|
||||
*block_len = 0;
|
||||
if (sep == NULL)
|
||||
return -1;
|
||||
*hdr_len = (size_t) ((const unsigned char *) sep - rec) + 4;
|
||||
cl = warc_memstr((const char *) rec, "Content-Length:", *hdr_len, 15);
|
||||
if (cl == NULL || sscanf(cl + 15, "%lld", block_len) != 1 ||
|
||||
*hdr_len + (size_t) *block_len + 4 != rlen)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* A cap-truncated body is still archived, tagged WARC-Truncated (v1.1). A
|
||||
compressed body cut short by a cap keeps its Content-Encoding (the stored
|
||||
bytes are the coded partial), so the record's label matches its body: assert
|
||||
the plaintext response carries "WARC-Truncated: length", and the gzip-coded
|
||||
one carries "WARC-Truncated: time", keeps Content-Encoding, and stores the
|
||||
coded bytes verbatim. */
|
||||
static int st_warc_trunc(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *end;
|
||||
int err = 0, trunc_len = 0, trunc_gz = 0, nresp = 0;
|
||||
static const char body[] = "partial body bytes\n";
|
||||
/* a valid gzip member (inflates to a known plaintext), as the coded partial
|
||||
*/
|
||||
static const unsigned char gz[] = {
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x0b, 0x2e,
|
||||
0x29, 0x4a, 0x2c, 0x49, 0x4d, 0xaf, 0xd4, 0x75, 0x54, 0x28, 0x4b, 0x2d,
|
||||
0x4a, 0x4a, 0x2c, 0xc9, 0xcc, 0x55, 0x08, 0x77, 0x0c, 0x72, 0x56, 0x48,
|
||||
0xca, 0x4f, 0xa9, 0xb4, 0x52, 0x28, 0xc9, 0x48, 0x55, 0x28, 0x2c, 0xcd,
|
||||
0x4c, 0xce, 0x56, 0x48, 0x2a, 0xca, 0x2f, 0xcf, 0x53, 0x48, 0xcb, 0xaf,
|
||||
0x50, 0xc8, 0x2a, 0xcd, 0x2d, 0x28, 0xd6, 0xe3, 0x02, 0x00, 0x5e, 0xb8,
|
||||
0xe7, 0x66, 0x3a, 0x00, 0x00, 0x00};
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-trunc: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-trunc.warc.gz");
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/big.bin", "127.0.0.1",
|
||||
"GET /big.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n", body,
|
||||
sizeof(body) - 1, NULL, 200, 0, WARC_TRUNC_LENGTH);
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/big.gz", "127.0.0.1",
|
||||
"GET /big.gz HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
|
||||
"gzip\r\n\r\n",
|
||||
(const char *) gz, sizeof(gz), NULL, 200, 0, WARC_TRUNC_TIME);
|
||||
warc_close(w);
|
||||
|
||||
data = warc_slurp(path, &data_len);
|
||||
assertf(data != NULL);
|
||||
p = data;
|
||||
end = data + data_len;
|
||||
while (p < end) {
|
||||
size_t rlen = 0, hdr_len = 0;
|
||||
long long block_len = 0;
|
||||
unsigned char *rec = warc_next_member(&p, end, &rlen);
|
||||
if (rec == NULL) {
|
||||
if (rlen != 0)
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0) {
|
||||
err = 1;
|
||||
freet(rec);
|
||||
continue;
|
||||
}
|
||||
if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) != NULL) {
|
||||
nresp++;
|
||||
if (warc_memstr((char *) rec,
|
||||
"WARC-Target-URI: http://test.local/big.bin", hdr_len,
|
||||
42) != NULL &&
|
||||
warc_memstr((char *) rec, "WARC-Truncated: length", hdr_len, 22) !=
|
||||
NULL)
|
||||
trunc_len = 1;
|
||||
if (warc_memstr((char *) rec, "WARC-Target-URI: http://test.local/big.gz",
|
||||
hdr_len, 41) != NULL) {
|
||||
const char *bsep = warc_memstr((char *) rec + hdr_len, "\r\n\r\n",
|
||||
(size_t) block_len, 4);
|
||||
size_t bodyoff = bsep ? (size_t) (bsep - (char *) rec) + 4 : 0;
|
||||
size_t got = bsep ? rlen - 4 - bodyoff : 0;
|
||||
/* WARC-Truncated: time, Content-Encoding kept, stored body == coded. */
|
||||
if (bsep != NULL &&
|
||||
warc_memstr((char *) rec, "WARC-Truncated: time", hdr_len, 20) !=
|
||||
NULL &&
|
||||
warc_memstr((char *) rec + hdr_len,
|
||||
"Content-Encoding:", (size_t) block_len, 17) != NULL &&
|
||||
got == sizeof(gz) && memcmp(rec + bodyoff, gz, sizeof(gz)) == 0)
|
||||
trunc_gz = 1;
|
||||
}
|
||||
}
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
if (!trunc_len || !trunc_gz || nresp != 2)
|
||||
err = 1;
|
||||
printf("warc-trunc: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* An ftp:// capture is ONE resource record: WARC-Type: resource, the payload's
|
||||
own Content-Type, block == payload, and no request/response pair. */
|
||||
static int st_warc_ftp(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *end;
|
||||
int err = 0, nresource = 0, nresp = 0, nreq = 0;
|
||||
static const char body[] = "\x00\x01"
|
||||
"FTP payload"
|
||||
"\x02\x03";
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-ftp: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-ftp.warc.gz");
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
warc_write_resource(w, "ftp://ftp.local/file.bin", "127.0.0.1",
|
||||
"application/octet-stream", body, sizeof(body) - 1, NULL,
|
||||
0);
|
||||
warc_close(w);
|
||||
|
||||
data = warc_slurp(path, &data_len);
|
||||
assertf(data != NULL);
|
||||
p = data;
|
||||
end = data + data_len;
|
||||
while (p < end) {
|
||||
size_t rlen = 0, hdr_len = 0;
|
||||
long long block_len = 0;
|
||||
unsigned char *rec = warc_next_member(&p, end, &rlen);
|
||||
if (rec == NULL) {
|
||||
if (rlen != 0)
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
|
||||
err = 1;
|
||||
if (warc_memstr((char *) rec, "WARC-Type: resource", hdr_len, 19) != NULL) {
|
||||
nresource++;
|
||||
if ((size_t) block_len != sizeof(body) - 1 ||
|
||||
memcmp(rec + hdr_len, body, sizeof(body) - 1) != 0)
|
||||
err = 1; /* block is the raw payload, no HTTP envelope */
|
||||
if (warc_memstr((char *) rec, "WARC-Target-URI: ftp://ftp.local/file.bin",
|
||||
hdr_len, 41) == NULL ||
|
||||
warc_memstr((char *) rec, "Content-Type: application/octet-stream",
|
||||
hdr_len, 38) == NULL)
|
||||
err = 1;
|
||||
}
|
||||
if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) != NULL)
|
||||
nresp++;
|
||||
if (warc_memstr((char *) rec, "WARC-Type: request", hdr_len, 18) != NULL)
|
||||
nreq++;
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
if (nresource != 1 || nresp != 0 || nreq != 0)
|
||||
err = 1;
|
||||
printf("warc-ftp: resource=%d response=%d request=%d: %s\n", nresource, nresp,
|
||||
nreq, err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* --warc-max-size rotates into <base>-00000.warc.gz, -00001, ...; each segment
|
||||
is independently valid and begins with its own warcinfo. */
|
||||
static int st_warc_rotate(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
char seg[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
LLint saved_max;
|
||||
unsigned char body[600];
|
||||
unsigned int rng = 0x12345678u;
|
||||
int err = 0, nseg = 0, i;
|
||||
size_t j;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-rotate: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
for (j = 0; j < sizeof(body);
|
||||
j++) { /* incompressible: gzip can't shrink it */
|
||||
rng ^= rng << 13;
|
||||
rng ^= rng >> 17;
|
||||
rng ^= rng << 5;
|
||||
body[j] = (unsigned char) (rng >> 24);
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-rot.warc.gz");
|
||||
saved_max = opt->warc_max_size;
|
||||
opt->warc_max_size =
|
||||
1000; /* a couple records per segment => several segments */
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
for (i = 0; i < 8; i++) {
|
||||
char uri[64];
|
||||
snprintf(uri, sizeof(uri), "http://test.local/f%d.bin", i);
|
||||
warc_write_transaction(
|
||||
w, uri, "127.0.0.1", "GET / HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n",
|
||||
(const char *) body, sizeof(body), NULL, 200, 0, 0);
|
||||
}
|
||||
warc_close(w);
|
||||
opt->warc_max_size = saved_max;
|
||||
|
||||
for (i = 0;; i++) {
|
||||
char fname[64];
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *pend;
|
||||
int first = 1;
|
||||
snprintf(fname, sizeof(fname), "warc-rot-%05d.warc.gz", i);
|
||||
fconcat(seg, sizeof(seg), argv[0], fname);
|
||||
data = warc_slurp(seg, &data_len);
|
||||
if (data == NULL)
|
||||
break; /* past the last segment */
|
||||
nseg++;
|
||||
p = data;
|
||||
pend = data + data_len;
|
||||
while (p < pend) {
|
||||
size_t rlen = 0, hdr_len = 0;
|
||||
long long block_len = 0;
|
||||
unsigned char *rec = warc_next_member(&p, pend, &rlen);
|
||||
if (rec == NULL) {
|
||||
if (rlen != 0)
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
|
||||
err = 1;
|
||||
if (first) { /* each segment leads with its own warcinfo */
|
||||
if (warc_memstr((char *) rec, "WARC-Type: warcinfo", hdr_len, 19) ==
|
||||
NULL)
|
||||
err = 1;
|
||||
first = 0;
|
||||
}
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
if (first) /* empty segment */
|
||||
err = 1;
|
||||
}
|
||||
if (nseg < 2)
|
||||
err = 1;
|
||||
printf("warc-rotate: %d segments: %s\n", nseg, err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* The default body storage: assert the stored WARC record is byte-verbatim gzip
|
||||
with Content-Encoding preserved and Content-Length = the coded length. */
|
||||
static int st_warc_verbatim(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *end;
|
||||
int err = 0, checked = 0;
|
||||
static const char a_plain[] =
|
||||
"Strategy-A verbatim WARC body: the quick brown fox jumps.\n";
|
||||
static const unsigned char a_gz[] = {
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x0b, 0x2e,
|
||||
0x29, 0x4a, 0x2c, 0x49, 0x4d, 0xaf, 0xd4, 0x75, 0x54, 0x28, 0x4b, 0x2d,
|
||||
0x4a, 0x4a, 0x2c, 0xc9, 0xcc, 0x55, 0x08, 0x77, 0x0c, 0x72, 0x56, 0x48,
|
||||
0xca, 0x4f, 0xa9, 0xb4, 0x52, 0x28, 0xc9, 0x48, 0x55, 0x28, 0x2c, 0xcd,
|
||||
0x4c, 0xce, 0x56, 0x48, 0x2a, 0xca, 0x2f, 0xcf, 0x53, 0x48, 0xcb, 0xaf,
|
||||
0x50, 0xc8, 0x2a, 0xcd, 0x2d, 0x28, 0xd6, 0xe3, 0x02, 0x00, 0x5e, 0xb8,
|
||||
0xe7, 0x66, 0x3a, 0x00, 0x00, 0x00};
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-verbatim: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-verbatim.warc.gz");
|
||||
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
/* the body is the coded (gzip) octets, stored verbatim. */
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/z.html", "127.0.0.1",
|
||||
"GET /z.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
|
||||
"gzip\r\nTransfer-Encoding: chunked\r\nContent-Length: 999\r\n\r\n",
|
||||
(const char *) a_gz, sizeof(a_gz), NULL, 200, 0, 0);
|
||||
warc_close(w);
|
||||
|
||||
data = warc_slurp(path, &data_len);
|
||||
assertf(data != NULL);
|
||||
p = data;
|
||||
end = data + data_len;
|
||||
|
||||
while (p < end) {
|
||||
size_t rlen = 0, hdr_len = 0;
|
||||
long long block_len = 0;
|
||||
unsigned char *rec = warc_next_member(&p, end, &rlen);
|
||||
if (rec == NULL) {
|
||||
if (rlen != 0)
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (warc_memstr((char *) rec, "msgtype=response", rlen, 16) == NULL) {
|
||||
freet(rec);
|
||||
continue;
|
||||
}
|
||||
if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0) {
|
||||
err = 1;
|
||||
freet(rec);
|
||||
continue;
|
||||
}
|
||||
/* Assert: one Content-Encoding, no Transfer-Encoding, Content-Length =
|
||||
compressed size. */
|
||||
{
|
||||
const char *block = (char *) rec + hdr_len;
|
||||
const char *ce =
|
||||
warc_memstr(block, "Content-Encoding:", (size_t) block_len, 17);
|
||||
const char *hcl =
|
||||
warc_memstr(block, "Content-Length:", (size_t) block_len, 15);
|
||||
long long http_cl = -1;
|
||||
int nce = 0;
|
||||
const char *scan = ce;
|
||||
while (scan != NULL) {
|
||||
size_t rem = (size_t) block_len - (size_t) (scan - block);
|
||||
nce++;
|
||||
scan = warc_memstr(scan + 17, "Content-Encoding:", rem - 17, 17);
|
||||
}
|
||||
if (ce == NULL || strncasecmp(ce + 17, " gzip", 5) != 0 || nce != 1)
|
||||
err = 1;
|
||||
if (warc_memstr(block, "Transfer-Encoding:", (size_t) block_len, 18) !=
|
||||
NULL)
|
||||
err = 1;
|
||||
if (hcl == NULL || sscanf(hcl + 15, "%lld", &http_cl) != 1 ||
|
||||
http_cl != (long long) sizeof(a_gz))
|
||||
err = 1;
|
||||
}
|
||||
/* Stored block bytes equal the gzip input, and inflate to the plaintext. */
|
||||
{
|
||||
const char *bsep = warc_memstr((char *) rec + hdr_len, "\r\n\r\n",
|
||||
(size_t) block_len, 4);
|
||||
if (bsep == NULL)
|
||||
err = 1;
|
||||
else {
|
||||
size_t bodyoff = (size_t) (bsep - (char *) rec) + 4;
|
||||
size_t got = rlen - 4 - bodyoff; /* minus the record trailer */
|
||||
if (got != sizeof(a_gz) ||
|
||||
memcmp(rec + bodyoff, a_gz, sizeof(a_gz)) != 0)
|
||||
err = 1;
|
||||
else {
|
||||
const unsigned char *bp = rec + bodyoff;
|
||||
size_t plen = 0;
|
||||
unsigned char *plain = warc_next_member(&bp, bp + got, &plen);
|
||||
if (plain == NULL || plen != sizeof(a_plain) - 1 ||
|
||||
memcmp(plain, a_plain, plen) != 0)
|
||||
err = 1;
|
||||
freet(plain);
|
||||
}
|
||||
}
|
||||
}
|
||||
checked = 1;
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
if (!checked)
|
||||
err = 1;
|
||||
printf("warc-verbatim: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* SURT canonicalization vectors (the CDXJ sort key: www-strip, default-port
|
||||
strip, host reversal, non-default port kept, IP/IPv6 verbatim). */
|
||||
static int st_warc_surt(httrackp *opt, int argc, char **argv) {
|
||||
static const struct {
|
||||
const char *url, *want;
|
||||
} cases[] = {
|
||||
{"http://www.example.com/", "com,example)/"},
|
||||
{"http://example.com:80/a/b?q=1", "com,example)/a/b?q=1"},
|
||||
{"https://www.EXAMPLE.com/Path", "com,example)/Path"},
|
||||
{"https://example.com:443/", "com,example)/"},
|
||||
{"http://www2.example.com/x", "com,example)/x"},
|
||||
{"http://example.com:8080/p", "com,example:8080)/p"},
|
||||
{"http://user:pass@www.example.com/y", "com,example)/y"},
|
||||
{"http://192.168.0.1/z", "192.168.0.1)/z"},
|
||||
{"http://[2001:db8::1]/w", "[2001:db8::1])/w"},
|
||||
{"http://sub.a.example.co.uk/deep?x=1#frag",
|
||||
"uk,co,example,a,sub)/deep?x=1"},
|
||||
};
|
||||
|
||||
int err = 0;
|
||||
size_t i;
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
char out[512];
|
||||
if (warc_surt(cases[i].url, out, sizeof(out)) != 0 ||
|
||||
strcmp(out, cases[i].want) != 0) {
|
||||
fprintf(stderr, "warc-surt: %s -> %s (want %s)\n", cases[i].url, out,
|
||||
cases[i].want);
|
||||
err = 1;
|
||||
}
|
||||
}
|
||||
printf("warc-surt: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* End-to-end CDXJ: crawl a handful of records with --warc-cdx, then verify the
|
||||
.cdx is sorted, has exactly one line per response/revisit/resource (none for
|
||||
warcinfo/request), and each offset/length points at a gzip member that
|
||||
independently inflates to a record whose WARC-Target-URI matches the line. */
|
||||
static int st_warc_cdx(httrackp *opt, int argc, char **argv) {
|
||||
char wpath[HTS_URLMAXSIZE], cpath[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
unsigned char *warc = NULL, *cdx = NULL;
|
||||
size_t warc_len = 0, cdx_len = 0;
|
||||
hts_boolean saved_cdx;
|
||||
int err = 0, nlines = 0;
|
||||
const char *lp, *cend;
|
||||
char prev[2048];
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-cdx: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(wpath, sizeof(wpath), argv[0], "warc-cdx.warc.gz");
|
||||
fconcat(cpath, sizeof(cpath), argv[0], "warc-cdx.cdx");
|
||||
saved_cdx = opt->warc_cdx;
|
||||
opt->warc_cdx = 1;
|
||||
w = warc_open(opt, wpath);
|
||||
assertf(w != NULL);
|
||||
warc_write_transaction(w, "http://www.example.com/one", "127.0.0.1",
|
||||
"GET /one HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
|
||||
"one body\n", 9, NULL, 200, 0, 0);
|
||||
warc_write_resource(w, "ftp://files.example.com/data.bin", "127.0.0.1",
|
||||
"application/octet-stream", "\x00\x01\x02\x03", 4, NULL,
|
||||
0);
|
||||
warc_write_transaction(w, "http://alpha.example.com/two", "127.0.0.1",
|
||||
"GET /two HTTP/1.1\r\nHost: alpha.example.com\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n",
|
||||
"two body\n", 9, NULL, 200, 0, 0);
|
||||
/* Same payload as /one at a new URL: identical-payload-digest revisit under
|
||||
OpenSSL, a full response otherwise; either way one index line. */
|
||||
warc_write_transaction(w, "http://zeta.example.com/dup", "127.0.0.1",
|
||||
"GET /dup HTTP/1.1\r\nHost: zeta.example.com\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
|
||||
"one body\n", 9, NULL, 200, 0, 0);
|
||||
warc_close(w);
|
||||
opt->warc_cdx = saved_cdx;
|
||||
|
||||
warc = warc_slurp(wpath, &warc_len);
|
||||
cdx = warc_slurp(cpath, &cdx_len);
|
||||
assertf(warc != NULL);
|
||||
assertf(cdx != NULL);
|
||||
|
||||
prev[0] = '\0';
|
||||
lp = (const char *) cdx;
|
||||
cend = (const char *) cdx + cdx_len;
|
||||
while (lp < cend) {
|
||||
const char *eol = memchr(lp, '\n', (size_t) (cend - lp));
|
||||
size_t llen = eol ? (size_t) (eol - lp) : (size_t) (cend - lp);
|
||||
char line[2048];
|
||||
const char *j, *us, *ue, *o, *l;
|
||||
char url[1024];
|
||||
unsigned long long off = 0, len = 0;
|
||||
const unsigned char *mp, *mend;
|
||||
unsigned char *rec;
|
||||
size_t rlen = 0, urllen;
|
||||
if (llen == 0) {
|
||||
lp = eol ? eol + 1 : cend;
|
||||
continue;
|
||||
}
|
||||
if (llen >= sizeof(line)) {
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
memcpy(line, lp, llen);
|
||||
line[llen] = '\0';
|
||||
nlines++;
|
||||
if (prev[0] != '\0' && strcmp(prev, line) > 0)
|
||||
err = 1; /* must be sorted */
|
||||
strlcpybuff(prev, line, sizeof(prev));
|
||||
j = strstr(line, "\"url\": \"");
|
||||
o = strstr(line, "\"offset\": \"");
|
||||
l = strstr(line, "\"length\": \"");
|
||||
if (j == NULL || o == NULL || l == NULL ||
|
||||
sscanf(o + 11, "%llu", &off) != 1 ||
|
||||
sscanf(l + 11, "%llu", &len) != 1) {
|
||||
err = 1;
|
||||
goto nextline;
|
||||
}
|
||||
us = j + 8;
|
||||
ue = strchr(us, '"');
|
||||
if (ue == NULL || (urllen = (size_t) (ue - us)) >= sizeof(url)) {
|
||||
err = 1;
|
||||
goto nextline;
|
||||
}
|
||||
memcpy(url, us, urllen);
|
||||
url[urllen] = '\0';
|
||||
if (len == 0 || off > warc_len || len > warc_len - off) {
|
||||
err = 1;
|
||||
goto nextline;
|
||||
}
|
||||
mp = warc + off;
|
||||
mend = warc + off + len;
|
||||
rec = warc_next_member(&mp, mend, &rlen);
|
||||
if (rec == NULL) {
|
||||
err = 1;
|
||||
goto nextline;
|
||||
}
|
||||
{
|
||||
char needle[1100];
|
||||
snprintf(needle, sizeof(needle), "WARC-Target-URI: %s\r\n", url);
|
||||
if (warc_memstr((char *) rec, needle, rlen, strlen(needle)) == NULL)
|
||||
err = 1;
|
||||
}
|
||||
freet(rec);
|
||||
nextline:
|
||||
lp = eol ? eol + 1 : cend;
|
||||
}
|
||||
freet(warc);
|
||||
freet(cdx);
|
||||
if (nlines != 4)
|
||||
err = 1; /* 3 responses/revisits + 1 resource; no warcinfo/request */
|
||||
printf("warc-cdx: %d index lines: %s\n", nlines, err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
/* Lowercase-hex SHA-256 of n bytes into out[65]; 1 on success. */
|
||||
static int wacz_test_sha256(const void *p, size_t n, char out[65]) {
|
||||
EVP_MD_CTX *c = EVP_MD_CTX_new();
|
||||
unsigned char md[EVP_MAX_MD_SIZE];
|
||||
unsigned int mdlen = 0, i;
|
||||
static const char hx[] = "0123456789abcdef";
|
||||
int ok;
|
||||
if (c == NULL)
|
||||
return 0;
|
||||
ok = EVP_DigestInit_ex(c, EVP_sha256(), NULL) == 1 &&
|
||||
(n == 0 || EVP_DigestUpdate(c, p, n) == 1) &&
|
||||
EVP_DigestFinal_ex(c, md, &mdlen) == 1 && mdlen == 32;
|
||||
EVP_MD_CTX_free(c);
|
||||
if (!ok)
|
||||
return 0;
|
||||
for (i = 0; i < 32; i++) {
|
||||
out[i * 2] = hx[md[i] >> 4];
|
||||
out[i * 2 + 1] = hx[md[i] & 0x0F];
|
||||
}
|
||||
out[64] = '\0';
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* One unzipped WACZ member: name, raw bytes, and the ZIP compression method. */
|
||||
typedef struct {
|
||||
char name[256];
|
||||
unsigned char *data;
|
||||
size_t len;
|
||||
int method;
|
||||
} wacz_entry;
|
||||
|
||||
/* Package a 2-record WARC as a WACZ, then unzip it in-process and assert the
|
||||
fixed layout, STORE-mode entries, recomputing sha256 digests, the digest
|
||||
chain, and the pages.jsonl header. */
|
||||
static int st_warc_wacz(httrackp *opt, int argc, char **argv) {
|
||||
char wpath[HTS_URLMAXSIZE], waczpath[HTS_URLMAXSIZE], cdxpath[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
hts_boolean saved_cdx, saved_wacz;
|
||||
wacz_entry ent[16];
|
||||
int nent = 0, err = 0, i;
|
||||
unzFile uf;
|
||||
const wacz_entry *dp = NULL, *dig = NULL, *pages = NULL;
|
||||
int have_archive = 0, have_index = 0, all_store = 1;
|
||||
LLint good_size;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-wacz: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(wpath, sizeof(wpath), argv[0], "warc-wacz.warc.gz");
|
||||
fconcat(waczpath, sizeof(waczpath), argv[0], "warc-wacz.wacz");
|
||||
fconcat(cdxpath, sizeof(cdxpath), argv[0], "warc-wacz.cdx");
|
||||
saved_cdx = opt->warc_cdx;
|
||||
saved_wacz = opt->warc_wacz;
|
||||
opt->warc_cdx = 1;
|
||||
opt->warc_wacz = 1;
|
||||
w = warc_open(opt, wpath);
|
||||
assertf(w != NULL);
|
||||
warc_write_transaction(w, "http://www.example.com/", "127.0.0.1",
|
||||
"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
|
||||
"<html>home</html>\n", 18, NULL, 200, 0, 0);
|
||||
warc_write_transaction(
|
||||
w, "http://www.example.com/data.bin", "127.0.0.1",
|
||||
"GET /data.bin HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\n"
|
||||
"Content-Type: application/octet-stream\r\n\r\n",
|
||||
"\x00\x01\x02\x03\x04", 5, NULL, 200, 0, 0);
|
||||
warc_close(w);
|
||||
|
||||
/* Unzip every member in-process. */
|
||||
uf = unzOpen(waczpath);
|
||||
assertf(uf != NULL);
|
||||
if (unzGoToFirstFile(uf) == UNZ_OK) {
|
||||
do {
|
||||
unz_file_info info;
|
||||
wacz_entry *e;
|
||||
if (nent >= (int) (sizeof(ent) / sizeof(ent[0]))) {
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
e = &ent[nent];
|
||||
if (unzGetCurrentFileInfo(uf, &info, e->name, sizeof(e->name), NULL, 0,
|
||||
NULL, 0) != UNZ_OK) {
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
e->method = (int) info.compression_method;
|
||||
e->len = (size_t) info.uncompressed_size;
|
||||
e->data = malloct(e->len ? e->len : 1);
|
||||
if (e->data == NULL || unzOpenCurrentFile(uf) != UNZ_OK) {
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (e->len > 0 &&
|
||||
unzReadCurrentFile(uf, e->data, (unsigned) e->len) != (int) e->len)
|
||||
err = 1;
|
||||
unzCloseCurrentFile(uf);
|
||||
nent++;
|
||||
} while (unzGoToNextFile(uf) == UNZ_OK);
|
||||
}
|
||||
unzClose(uf);
|
||||
|
||||
/* Classify members and assert STORE mode (WACZ spec requirement). */
|
||||
for (i = 0; i < nent; i++) {
|
||||
const wacz_entry *e = &ent[i];
|
||||
if (e->method != 0)
|
||||
all_store = 0;
|
||||
if (strncmp(e->name, "archive/", 8) == 0)
|
||||
have_archive = 1;
|
||||
else if (strcmp(e->name, "indexes/index.cdx") == 0)
|
||||
have_index = 1;
|
||||
else if (strcmp(e->name, "pages/pages.jsonl") == 0)
|
||||
pages = e;
|
||||
else if (strcmp(e->name, "datapackage.json") == 0)
|
||||
dp = e;
|
||||
else if (strcmp(e->name, "datapackage-digest.json") == 0)
|
||||
dig = e;
|
||||
}
|
||||
if (!have_archive || !have_index || pages == NULL || dp == NULL ||
|
||||
dig == NULL || !all_store)
|
||||
err = 1;
|
||||
|
||||
/* pages.jsonl: header line, then >= 1 body row carrying url + ts. */
|
||||
if (pages != NULL) {
|
||||
if (pages->len < 27 ||
|
||||
memcmp(pages->data, "{\"format\": \"json-pages-1.0\"", 27) != 0)
|
||||
err = 1;
|
||||
else {
|
||||
const char *nl = memchr(pages->data, '\n', pages->len);
|
||||
const char *body = nl ? nl + 1 : NULL;
|
||||
size_t blen =
|
||||
body ? pages->len - (size_t) (body - (char *) pages->data) : 0;
|
||||
if (body == NULL || blen == 0 ||
|
||||
warc_memstr(body, "\"url\": ", blen, 7) == NULL ||
|
||||
warc_memstr(body, "\"ts\": ", blen, 6) == NULL)
|
||||
err = 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Every datapackage resource hash recomputes from the stored member bytes. */
|
||||
if (dp != NULL) {
|
||||
char *json = malloct(dp->len + 1);
|
||||
if (json == NULL) {
|
||||
err = 1;
|
||||
} else {
|
||||
const char *p;
|
||||
memcpy(json, dp->data, dp->len);
|
||||
json[dp->len] = '\0';
|
||||
if (strstr(json, "\"profile\": \"data-package\"") == NULL ||
|
||||
strstr(json, "\"wacz_version\": \"") == NULL)
|
||||
err = 1;
|
||||
p = json;
|
||||
while ((p = strstr(p, "\"path\": \"")) != NULL) {
|
||||
char path[256], want[80], got[65];
|
||||
const char *pe, *h;
|
||||
size_t plen;
|
||||
p += 9;
|
||||
pe = strchr(p, '"');
|
||||
if (pe == NULL || (plen = (size_t) (pe - p)) >= sizeof(path)) {
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
memcpy(path, p, plen);
|
||||
path[plen] = '\0';
|
||||
h = strstr(pe, "\"hash\": \"sha256:");
|
||||
if (h == NULL || sscanf(h + 16, "%79[0-9a-f]", want) != 1) {
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
for (i = 0; i < nent; i++)
|
||||
if (strcmp(ent[i].name, path) == 0)
|
||||
break;
|
||||
if (i == nent || !wacz_test_sha256(ent[i].data, ent[i].len, got) ||
|
||||
strcmp(got, want) != 0)
|
||||
err = 1;
|
||||
p = pe;
|
||||
}
|
||||
freet(json);
|
||||
}
|
||||
}
|
||||
|
||||
/* datapackage-digest.json chains sha256(datapackage.json). */
|
||||
if (dp != NULL && dig != NULL) {
|
||||
char dphex[65], *djson = malloct(dig->len + 1);
|
||||
const char *h;
|
||||
char want[80];
|
||||
if (djson == NULL || !wacz_test_sha256(dp->data, dp->len, dphex)) {
|
||||
err = 1;
|
||||
} else {
|
||||
memcpy(djson, dig->data, dig->len);
|
||||
djson[dig->len] = '\0';
|
||||
if (strstr(djson, "\"path\": \"datapackage.json\"") == NULL)
|
||||
err = 1;
|
||||
h = strstr(djson, "\"hash\": \"sha256:");
|
||||
if (h == NULL || sscanf(h + 16, "%79[0-9a-f]", want) != 1 ||
|
||||
strcmp(want, dphex) != 0)
|
||||
err = 1;
|
||||
}
|
||||
freet(djson);
|
||||
}
|
||||
|
||||
for (i = 0; i < nent; i++)
|
||||
freet(ent[i].data);
|
||||
|
||||
/* #522-class: a failed re-package must leave the existing .wacz untouched.
|
||||
Drop the .cdx and re-run empty so packaging fails on the missing index. */
|
||||
good_size = fsize(waczpath);
|
||||
if (good_size <= 0)
|
||||
err = 1;
|
||||
(void) UNLINK(cdxpath);
|
||||
w = warc_open(opt, wpath);
|
||||
assertf(w != NULL);
|
||||
warc_close(w);
|
||||
if (fsize(waczpath) != good_size) /* destroyed or rewritten = data loss */
|
||||
err = 1;
|
||||
|
||||
opt->warc_cdx = saved_cdx;
|
||||
opt->warc_wacz = saved_wacz;
|
||||
printf("warc-wacz: %d members (store=%d): %s\n", nent, all_store,
|
||||
err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* Registry: name -> handler, with a usage hint and a one-line description. */
|
||||
/* ------------------------------------------------------------ */
|
||||
@@ -4425,22 +3655,6 @@ static const struct selftest_entry {
|
||||
{"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
|
||||
{"warc", "<dir>", "WARC/1.1 writer: framing, digests, revisit dedup",
|
||||
st_warc},
|
||||
{"warc-trunc", "<dir>", "WARC-Truncated on a cap-truncated body",
|
||||
st_warc_trunc},
|
||||
{"warc-ftp", "<dir>", "ftp resource record (no HTTP envelope)",
|
||||
st_warc_ftp},
|
||||
{"warc-rotate", "<dir>", "--warc-max-size segment rotation",
|
||||
st_warc_rotate},
|
||||
{"warc-verbatim", "<dir>", "verbatim compressed response body (default)",
|
||||
st_warc_verbatim},
|
||||
{"warc-surt", "", "SURT canonicalization of the CDXJ sort key",
|
||||
st_warc_surt},
|
||||
{"warc-cdx", "<dir>", "--warc-cdx CDXJ index: sorted, offsets inflate",
|
||||
st_warc_cdx},
|
||||
#if HTS_USEOPENSSL
|
||||
{"warc-wacz", "<dir>", "--wacz package: layout, STORE mode, sha256 digests",
|
||||
st_warc_wacz},
|
||||
#endif
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
1057
src/htswarc.c
1057
src/htswarc.c
File diff suppressed because it is too large
Load Diff
@@ -45,16 +45,6 @@ extern "C" {
|
||||
under the project's output directory at open time. */
|
||||
#define WARC_AUTONAME "\001auto"
|
||||
|
||||
/* htsblk.warc_truncated / WARC-Truncated reason tokens (ISO 28500 sec 5.13).
|
||||
A cap-truncated body is still archived, tagged with why it was cut short. */
|
||||
#define WARC_TRUNC_NONE 0
|
||||
#define WARC_TRUNC_LENGTH 1 /* hit a size cap (-M mirror / -m per-file) */
|
||||
#define WARC_TRUNC_TIME 2 /* hit the mirror time cap (-E/--max-time) */
|
||||
#define WARC_TRUNC_DISCONNECT 3 /* connection dropped mid-body */
|
||||
|
||||
/* WARC-Truncated token for a warc_truncated code, or NULL for none. */
|
||||
const char *warc_truncated_reason(int code);
|
||||
|
||||
typedef struct warc_writer warc_writer;
|
||||
|
||||
/* Stash the raw request header block (bstr.buffer) on r for the later WARC
|
||||
@@ -68,13 +58,6 @@ void warc_stash_response(htsblk *r, const char *resphdr);
|
||||
/* Free both stashed header blocks (idempotent, NULL-safe). */
|
||||
void warc_free_request(htsblk *r);
|
||||
|
||||
/* Adopt the de-chunked compressed spool at tmpfile_path onto
|
||||
r->warc_rawpath/warc_rawsize (strdupt; frees any prior) so the WARC record
|
||||
stores the body verbatim. No-op leaving warc_rawpath NULL when tmpfile_path
|
||||
is empty or the spool is missing/empty (the record then stores the decoded
|
||||
in-memory/on-disk body instead). */
|
||||
void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path);
|
||||
|
||||
/* Emit the request + response (or revisit) records for one finished
|
||||
transaction. Lazily opens the writer into opt->state.warc; a no-op (logged
|
||||
once) if the archive cannot be created. */
|
||||
@@ -93,10 +76,6 @@ warc_writer *warc_open(httrackp *opt, const char *path);
|
||||
/* Flush, close and free the writer (NULL-safe). */
|
||||
void warc_close(warc_writer *w);
|
||||
|
||||
/* SURT-canonicalize url into out[outsz] (the CDXJ sort key). Returns 0 on
|
||||
success, -1 on error or truncation. Exposed for the -#test=warc-surt test. */
|
||||
int warc_surt(const char *url, char *out, size_t outsz);
|
||||
|
||||
/* Write one transaction's request + response (or revisit) records.
|
||||
target_uri: absolute URL fetched.
|
||||
ip: numeric peer IP, or NULL/"" to omit.
|
||||
@@ -105,24 +84,12 @@ int warc_surt(const char *url, char *out, size_t outsz);
|
||||
body/body_len: decoded in-memory body, or NULL when on disk.
|
||||
body_path: file re-read for the body when body==NULL (may be NULL).
|
||||
is_update_unchanged: nonzero for a 304 server-not-modified revisit.
|
||||
truncated: a WARC_TRUNC_* reason to tag a cap-truncated body, else 0.
|
||||
The body is stored verbatim: Content-Encoding is kept and Content-Length set
|
||||
to body_len, so body/body_len must be the as-received (coded) bytes.
|
||||
Returns 0 on success, -1 on error. */
|
||||
int warc_write_transaction(warc_writer *w, const char *target_uri,
|
||||
const char *ip, const char *req_hdr,
|
||||
const char *resp_hdr, const char *body,
|
||||
size_t body_len, const char *body_path,
|
||||
int statuscode, int is_update_unchanged,
|
||||
int truncated);
|
||||
|
||||
/* Write one non-HTTP capture as a single WARC 'resource' record: the block is
|
||||
the raw payload (no HTTP envelope), Content-Type is the payload's own MIME.
|
||||
Used for ftp:// transfers. truncated is a WARC_TRUNC_* reason or 0.
|
||||
Returns 0 on success, -1 on error. */
|
||||
int warc_write_resource(warc_writer *w, const char *target_uri, const char *ip,
|
||||
const char *content_type, const char *body,
|
||||
size_t body_len, const char *body_path, int truncated);
|
||||
int statuscode, int is_update_unchanged);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# SURT canonicalization of the CDXJ sort key (--warc-cdx). Pure string work,
|
||||
# so it runs under the MSan-instrumented 01_engine glob.
|
||||
httrack -O /dev/null -#test=warc-surt | grep -q "warc-surt: OK"
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --warc-cdx CDXJ index over synthetic transactions: sorted, one line per
|
||||
# response/revisit/resource, each offset/length inflates to the right member.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc-cdx "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --wacz packaging over synthetic transactions: the WACZ unzips in-process to
|
||||
# the fixed layout, every entry is ZIP STORE, each datapackage sha256 recomputes
|
||||
# from the stored bytes, and the digest chains datapackage.json. WACZ needs the
|
||||
# OpenSSL SHA-256 digests, so the self-test is absent on non-OpenSSL builds.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
if ! "$httrack_bin" -#test 2>&1 | grep -q '^ warc-wacz'; then
|
||||
echo "warc-wacz self-test unavailable (build without OpenSSL); skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc-wacz "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
@@ -4,19 +4,16 @@
|
||||
set -euo pipefail
|
||||
|
||||
# WARC/1.1 writer self-test: framing, Content-Length, gzip members, round-trip,
|
||||
# revisit dedup, plus v1.1 WARC-Truncated, ftp resource records, and
|
||||
# --warc-max-size rotation, over synthetic transactions.
|
||||
# and identical-payload-digest revisit dedup, over synthetic transactions.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
for t in warc warc-trunc warc-ftp warc-rotate warc-verbatim; do
|
||||
out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
done
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A --warc crawl stores compressed bodies verbatim (the default): the response
|
||||
# record keeps Content-Encoding: gzip and its stored bytes inflate back to the
|
||||
# served page. The validator's --verbatim mode is the differential gate:
|
||||
# inflate(stored) must equal the decoded body the server compressed.
|
||||
#
|
||||
# Two fixtures cover both adoption branches of back_finalize: page.html (text/html)
|
||||
# takes the in-memory branch; data.bin (application/octet-stream) is streamed to
|
||||
# disk, so it exercises the is_write direct-to-disk spool adoption.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# decoded bodies served (gzip-coded) by route_warcgz_page and route_warcgz_data.
|
||||
export WARC_VALIDATE_BODY="warcgz/page.html=3c68746d6c3e3c626f64793e766572626174696d20677a6970207061676520666f72205741524320737472617465677920413c2f626f64793e3c2f68746d6c3e0a warcgz/data.bin=766572626174696d20677a6970206f637465742d73747265616d20626f647920666f722074686520574152432069735f777269746520706174680a"
|
||||
export WARC_VALIDATE_VERBATIM=1
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --warc-validate \
|
||||
--found 'warcgz/index.html' --found 'warcgz/page.html' --found 'warcgz/data.bin' \
|
||||
httrack 'BASEURL/warcgz/index.html' --warc-file warc-out
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A --wacz crawl packages the WARC archive, its CDXJ index and a generated
|
||||
# pages.jsonl into a single WACZ file at crawl end. The stdlib validator is the
|
||||
# real gate: STORE-mode entries, the fixed layout, recomputed sha256 digests and
|
||||
# the datapackage-digest chain (plus py-wacz/pywb when importable). The crawl
|
||||
# skips cleanly on a build without OpenSSL (no conformant SHA-256 -> no package).
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --wacz-validate \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --wacz
|
||||
@@ -3,7 +3,7 @@
|
||||
# silently drop it from the dist tarball and break "make distcheck".
|
||||
EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
proxy-https-server.py socks5-server.py proxy-connect-server.py \
|
||||
proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \
|
||||
proxytestlib.py tls-stall-server.py warc-validate.py \
|
||||
local-crawl.sh local-server.py testlib.sh server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
@@ -78,12 +78,9 @@ TESTS = \
|
||||
01_engine-unescape-bounds.test \
|
||||
01_engine-useragent.test \
|
||||
01_engine-version-macros.test \
|
||||
01_engine-warc-surt.test \
|
||||
01_engine-xfread.test \
|
||||
01_zlib-acceptencoding.test \
|
||||
01_zlib-warc.test \
|
||||
01_zlib-warc-cdx.test \
|
||||
01_zlib-warc-wacz.test \
|
||||
01_zlib-contentcodings.test \
|
||||
01_zlib-cache.test \
|
||||
01_zlib-cache-corrupt.test \
|
||||
@@ -160,8 +157,6 @@ TESTS = \
|
||||
69_local-intl-logdir.test \
|
||||
71_local-crange-repaircache.test \
|
||||
72_watchdog-crawl.test \
|
||||
73_local-warc.test \
|
||||
74_local-warc-wacz.test \
|
||||
74_local-warc-verbatim.test
|
||||
73_local-warc.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -48,7 +48,6 @@ key="${testdir}/server.key"
|
||||
tls=
|
||||
verbose=
|
||||
warc_validate=
|
||||
wacz_validate=
|
||||
html_subdir=
|
||||
outdir_intl=
|
||||
rerun=
|
||||
@@ -118,8 +117,6 @@ while test "$pos" -lt "$nargs"; do
|
||||
--rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
|
||||
# validate the produced .warc.gz (see the validation block near the end)
|
||||
--warc-validate) warc_validate=1 ;;
|
||||
# validate the produced .wacz package (stdlib, plus py-wacz/pywb if present)
|
||||
--wacz-validate) wacz_validate=1 ;;
|
||||
--no-purge)
|
||||
nopurge=1
|
||||
audit+=("--no-purge")
|
||||
@@ -375,13 +372,9 @@ if test -n "$warc_validate"; then
|
||||
fresh="${tmpdir}/warc-pass1.gz"
|
||||
test -f "$fresh" || fresh="$warc"
|
||||
declare -a bodyargs=()
|
||||
# WARC_VALIDATE_BODY holds one or more whitespace-separated SUB=HEX specs.
|
||||
for spec in ${WARC_VALIDATE_BODY:-}; do
|
||||
bodyargs+=(--expect-body-hex "$spec")
|
||||
done
|
||||
# compressed asset: assert the stored (verbatim) body inflates to the served
|
||||
# body and keeps Content-Encoding, instead of expecting a decoded body.
|
||||
test -n "${WARC_VALIDATE_VERBATIM:-}" && bodyargs+=(--verbatim)
|
||||
if test -n "${WARC_VALIDATE_BODY:-}"; then
|
||||
bodyargs=(--expect-body-hex "$WARC_VALIDATE_BODY")
|
||||
fi
|
||||
info "validating fresh WARC (response bodies)"
|
||||
"$python" "$validator" "$(nativepath "$fresh")" "${bodyargs[@]}" >&2 ||
|
||||
die "fresh WARC validation failed"
|
||||
@@ -405,23 +398,6 @@ if test -n "$warc_validate"; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- optional WACZ validation (--wacz) --------------------------------------
|
||||
if test -n "$wacz_validate"; then
|
||||
wacz=$(find "$mirrorroot" -maxdepth 2 -name '*.wacz' 2>/dev/null | sort | tail -n1)
|
||||
if test -z "$wacz"; then
|
||||
# No package: only acceptable when the build lacks OpenSSL (SHA-256).
|
||||
if grep -aqi "WACZ requires an OpenSSL" "${logroot}/hts-log.txt"; then
|
||||
info "no .wacz produced (build without OpenSSL); skipping"
|
||||
exit 77
|
||||
fi
|
||||
die "no .wacz file produced under $mirrorroot"
|
||||
fi
|
||||
validator=$(nativepath "${testdir}/wacz-validate.py")
|
||||
info "validating WACZ package"
|
||||
"$python" "$validator" "$(nativepath "$wacz")" >&2 || die "WACZ validation failed"
|
||||
result "OK"
|
||||
fi
|
||||
|
||||
# No crawl, even a cancelled one, may leave engine temporaries: .delayed (#107,
|
||||
# #483), or the .z/.u content-coding temps (#557).
|
||||
info "checking for leftover engine temporaries"
|
||||
|
||||
@@ -635,34 +635,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
extra_headers=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
|
||||
# A gzip-coded HTML page whose decoded body is known, for the verbatim-WARC
|
||||
# differential (stored compressed bytes must inflate to this).
|
||||
WARCGZ_BODY = b"<html><body>verbatim gzip page for WARC strategy A</body></html>\n"
|
||||
|
||||
# A NON-html gzip-coded asset: HTTrack streams it straight to disk, so the
|
||||
# verbatim spool adoption runs on the is_write (direct-to-disk) branch of
|
||||
# back_finalize, not the in-memory branch route_warcgz_page exercises.
|
||||
WARCGZ_BIN_BODY = b"verbatim gzip octet-stream body for the WARC is_write path\n"
|
||||
|
||||
def route_warcgz_index(self):
|
||||
self.send_html(
|
||||
'\t<a href="page.html">page</a>\n' '\t<a href="data.bin">data</a>\n'
|
||||
)
|
||||
|
||||
def route_warcgz_page(self):
|
||||
self.send_raw(
|
||||
gzip.compress(self.WARCGZ_BODY),
|
||||
"text/html",
|
||||
extra_headers=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
|
||||
def route_warcgz_data(self):
|
||||
self.send_raw(
|
||||
gzip.compress(self.WARCGZ_BIN_BODY),
|
||||
"application/octet-stream",
|
||||
extra_headers=[("Content-Encoding", "gzip")],
|
||||
)
|
||||
|
||||
# --- content codings ---------------------------------------------------
|
||||
# Canned br/zstd bodies (no brotli/zstd module in the stdlib): both decode
|
||||
# to CODEC_BODY. Regenerate with the brotli/zstd CLIs over that string.
|
||||
@@ -1570,9 +1542,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/gated/index.php": route_gated_index,
|
||||
"/gated/secret.php": route_gated_secret,
|
||||
"/robots.txt": route_robots,
|
||||
"/warcgz/index.html": route_warcgz_index,
|
||||
"/warcgz/page.html": route_warcgz_page,
|
||||
"/warcgz/data.bin": route_warcgz_data,
|
||||
"/codec/index.html": route_codec_index,
|
||||
"/codec/br.html": route_codec_br,
|
||||
"/codec/zstd.html": route_codec_zstd,
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Validate a WACZ package with the stdlib only (no py-wacz needed): every entry
|
||||
# is ZIP STORE, the fixed layout is present, each datapackage resource hash and
|
||||
# size recomputes, the digest chains datapackage.json, and pages.jsonl carries
|
||||
# the json-pages-1.0 header. If py-wacz (`wacz validate`) is importable it runs
|
||||
# too; both gates must pass. Exit 0 = valid, nonzero = invalid.
|
||||
import sys
|
||||
import json
|
||||
import zipfile
|
||||
import hashlib
|
||||
|
||||
|
||||
def fail(msg):
|
||||
print("wacz-validate: FAIL: %s" % msg, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
fail("usage: wacz-validate.py FILE.wacz")
|
||||
path = sys.argv[1]
|
||||
z = zipfile.ZipFile(path)
|
||||
names = z.namelist()
|
||||
|
||||
for info in z.infolist():
|
||||
if info.compress_type != zipfile.ZIP_STORED:
|
||||
fail("%s is not STORE mode (%d)" % (info.filename, info.compress_type))
|
||||
|
||||
need_arc = any(
|
||||
n.startswith("archive/") and n.endswith((".warc.gz", ".warc")) for n in names
|
||||
)
|
||||
for req, ok in (
|
||||
("archive/*.warc.gz", need_arc),
|
||||
("indexes/index.cdx", "indexes/index.cdx" in names),
|
||||
("pages/pages.jsonl", "pages/pages.jsonl" in names),
|
||||
("datapackage.json", "datapackage.json" in names),
|
||||
("datapackage-digest.json", "datapackage-digest.json" in names),
|
||||
):
|
||||
if not ok:
|
||||
fail("missing %s (entries: %s)" % (req, names))
|
||||
|
||||
lines = [ln for ln in z.read("pages/pages.jsonl").split(b"\n") if ln.strip()]
|
||||
if not lines or json.loads(lines[0]).get("format") != "json-pages-1.0":
|
||||
fail("pages.jsonl header is not json-pages-1.0: %r" % lines[:1])
|
||||
body = [json.loads(ln) for ln in lines[1:]]
|
||||
if not body:
|
||||
fail("pages.jsonl has no page rows after the header")
|
||||
for row in body:
|
||||
if "url" not in row or "ts" not in row:
|
||||
fail("pages.jsonl row missing url/ts: %r" % row)
|
||||
|
||||
dp = json.loads(z.read("datapackage.json"))
|
||||
if dp.get("profile") != "data-package":
|
||||
fail("profile != data-package: %r" % dp.get("profile"))
|
||||
if dp.get("wacz_version") != "1.1.1":
|
||||
fail("wacz_version != 1.1.1: %r" % dp.get("wacz_version"))
|
||||
resources = dp.get("resources", [])
|
||||
if not resources:
|
||||
fail("datapackage has no resources")
|
||||
for r in resources:
|
||||
data = z.read(r["path"])
|
||||
h = "sha256:" + hashlib.sha256(data).hexdigest()
|
||||
if h != r["hash"]:
|
||||
fail("%s hash %s != %s" % (r["path"], h, r["hash"]))
|
||||
if len(data) != r["bytes"]:
|
||||
fail("%s bytes %d != %d" % (r["path"], len(data), r["bytes"]))
|
||||
|
||||
dig = json.loads(z.read("datapackage-digest.json"))
|
||||
want = "sha256:" + hashlib.sha256(z.read("datapackage.json")).hexdigest()
|
||||
if dig.get("path") != "datapackage.json" or dig.get("hash") != want:
|
||||
fail("digest chain broken: %r" % dig)
|
||||
|
||||
print(
|
||||
"wacz-validate: OK (%d entries, %d resources, stdlib)"
|
||||
% (len(names), len(resources))
|
||||
)
|
||||
|
||||
# Optional stricter gate when py-wacz is present.
|
||||
try:
|
||||
import wacz # noqa: F401
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
from wacz.main import main as wacz_main # type: ignore
|
||||
except Exception:
|
||||
return
|
||||
print("wacz-validate: running py-wacz validate")
|
||||
rc = wacz_main(["validate", "-f", path])
|
||||
if rc not in (0, None):
|
||||
fail("py-wacz validate returned %r" % rc)
|
||||
print("wacz-validate: py-wacz OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -11,11 +11,6 @@
|
||||
# WARC-Payload-Digest matching sha1(body) when present
|
||||
# --no-response-for SUB the asset containing SUB must be a revisit: no
|
||||
# response may target it, and a revisit must
|
||||
# --verbatim compressed asset: --expect-body-hex instead keeps
|
||||
# Content-Encoding, checks the HTTP Content-Length is
|
||||
# the stored (compressed) length, asserts the stored
|
||||
# body inflates to HEX (the served plaintext), and
|
||||
# requires the payload digest when the file emits any
|
||||
import base64
|
||||
import hashlib
|
||||
import sys
|
||||
@@ -66,53 +61,14 @@ def check_body(rec, http_hdr, body, sub, want):
|
||||
sys.exit("WARC-Payload-Digest mismatch for %s" % sub)
|
||||
|
||||
|
||||
def check_body_verbatim(rec, http_hdr, body, sub, want, digests_emitted):
|
||||
"""Verbatim: the stored body is the coded octets, Content-Encoding is kept,
|
||||
the HTTP Content-Length equals the stored (compressed) length, inflating the
|
||||
body yields the served plaintext, and the payload digest is over the coded
|
||||
body. The differential: inflate(stored) == the body the server compressed."""
|
||||
if b"Content-Encoding" not in http_hdr:
|
||||
sys.exit("verbatim record for %s dropped Content-Encoding" % sub)
|
||||
if b"Transfer-Encoding" in http_hdr:
|
||||
sys.exit("verbatim record for %s kept Transfer-Encoding" % sub)
|
||||
hcl = field(http_hdr, b"Content-Length")
|
||||
if hcl is None or int(hcl) != len(body):
|
||||
sys.exit("verbatim record for %s: HTTP Content-Length != stored body" % sub)
|
||||
try:
|
||||
decoded = zlib.decompress(body, zlib.MAX_WBITS | 16)
|
||||
except Exception as exc:
|
||||
sys.exit("verbatim record for %s: body did not inflate: %s" % (sub, exc))
|
||||
if decoded != want:
|
||||
sys.exit(
|
||||
"verbatim decoded mismatch for %s: got %d bytes, expected %d"
|
||||
% (sub, len(decoded), len(want))
|
||||
)
|
||||
pd = field(rec[: rec.find(b"\r\n\r\n")], b"WARC-Payload-Digest")
|
||||
# Catch a regression that drops the digest on the verbatim path, but only
|
||||
# when this file emits digests at all (an OpenSSL build; none otherwise).
|
||||
if digests_emitted and pd is None:
|
||||
sys.exit("verbatim record for %s: missing WARC-Payload-Digest" % sub)
|
||||
if pd is not None and pd.startswith(b"sha1:"):
|
||||
want_b32 = base64.b32encode(hashlib.sha1(body).digest()).decode("ascii")
|
||||
if pd[5:].decode("ascii") != want_b32:
|
||||
sys.exit("WARC-Payload-Digest (compressed) mismatch for %s" % sub)
|
||||
|
||||
|
||||
def main():
|
||||
argv = sys.argv[1:]
|
||||
expect_revisit = "--expect-revisit" in argv
|
||||
verbatim = "--verbatim" in argv
|
||||
body_specs = [s.split("=", 1) for s in opt_values(argv, "--expect-body-hex")]
|
||||
no_resp = opt_values(argv, "--no-response-for")
|
||||
path = [a for a in argv if not a.startswith("--") and "=" not in a][0]
|
||||
data = open(path, "rb").read()
|
||||
|
||||
# digests are emitted only on an OpenSSL build; detect it once so --verbatim
|
||||
# can require the payload digest exactly when the file carries any.
|
||||
digests_emitted = any(
|
||||
b"WARC-Payload-Digest" in r[: r.find(b"\r\n\r\n")] for r in records(data)
|
||||
)
|
||||
|
||||
total = revisits = responses = infos = 0
|
||||
body_hits = {sub: False for sub, _ in body_specs}
|
||||
revisit_hits = {sub: False for sub in no_resp}
|
||||
@@ -150,13 +106,7 @@ def main():
|
||||
http_hdr, body = block[:bsep], block[bsep + 4 :]
|
||||
for sub, hexval in body_specs:
|
||||
if sub.encode() in uri:
|
||||
want = bytes.fromhex(hexval)
|
||||
if verbatim:
|
||||
check_body_verbatim(
|
||||
rec, http_hdr, body, sub, want, digests_emitted
|
||||
)
|
||||
else:
|
||||
check_body(rec, http_hdr, body, sub, want)
|
||||
check_body(rec, http_hdr, body, sub, bytes.fromhex(hexval))
|
||||
body_hits[sub] = True
|
||||
elif wtype == b"revisit":
|
||||
revisits += 1
|
||||
|
||||
Reference in New Issue
Block a user