mirror of
https://github.com/xroche/httrack.git
synced 2026-07-26 18:49:39 +03:00
Compare commits
1 Commits
pt-readfie
...
feat/singl
| Author | SHA1 | Date | |
|---|---|---|---|
|
b90d699d2e |
@@ -300,6 +300,7 @@ ones it kept so the local copy browses offline. These options tune both halves.<
|
||||
<tr><td><tt>--preserve (-%p), --disable-passwords (-%x)</tt></td><td>Leave HTML untouched (no rewriting), and strip passwords out of saved links.</td></tr>
|
||||
<tr><td><tt>--extended-parsing (-%P), --parse-java (-j)</tt></td><td>Aggressive link discovery, and how much script content is parsed for links.</td></tr>
|
||||
<tr><td><tt>--mime-html (-%M)</tt></td><td>Save the whole mirror as a single MIME-encapsulated <tt>.mht</tt> archive (<tt>index.mht</tt>).</td></tr>
|
||||
<tr><td><tt>--single-file (-%Z), --single-file-max-size N</tt></td><td>Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded as <tt>data:</tt> URIs. Assets over the cap (10 MB by default) keep their link, as do audio, video, and the links from one page to another.</td></tr>
|
||||
<tr><td><tt>--index (-I), --build-top-index (-%i), --search-index (-%I)</tt></td><td>Build a per-mirror index, a top index across projects, and a searchable keyword index.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -482,6 +483,18 @@ add <tt>--warc-cdx</tt> for a sorted CDXJ index, or <tt>--wacz</tt> to bundle th
|
||||
archive, index and pages into one WACZ for replay tools such as
|
||||
replayweb.page.</small></p>
|
||||
|
||||
<h4>Pages you can hand to someone as one file</h4>
|
||||
<p><tt>httrack https://example.com/ --single-file --path mydir</tt><br>
|
||||
<small>Rewrites each saved page after the crawl so its stylesheets, scripts,
|
||||
images and fonts are embedded as <tt>data:</tt> URIs. The mirror stays a normal
|
||||
browsable tree — links between pages remain relative, and the assets are
|
||||
still on disk — but any single <tt>.html</tt> file can now be mailed or
|
||||
archived on its own. Unlike <tt>--mime-html</tt>, which produces one
|
||||
<tt>.mht</tt> archive that current browsers no longer open, the output is
|
||||
ordinary HTML. Raise or lower the 10 MB per-asset limit with
|
||||
<tt>--single-file-max-size N</tt>; anything over it, plus audio and video, keeps
|
||||
its link.</small></p>
|
||||
|
||||
<h4>HTTrack as a fetch tool</h4>
|
||||
<p><tt>httrack --get https://host/file.bin --path tmp</tt><br>
|
||||
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
|
||||
|
||||
@@ -90,9 +90,10 @@ offline browser : copy websites to a local directory</p>
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
<b>-%x, --disable-passwords</b> ] [ <b>-%q,
|
||||
] [ <b>-%Z, --single-file</b> ] [ <b>-LN,
|
||||
--long-names[=N]</b> ] [ <b>-KN, --keep-links[=N]</b> ] [
|
||||
<b>-x, --replace-external</b> ] [ <b>-%x,
|
||||
--disable-passwords</b> ] [ <b>-%q,
|
||||
--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
|
||||
<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
|
||||
] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
|
||||
@@ -648,6 +649,22 @@ don’t wait) (--cached-delayed-type-check)</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%Z</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>after the mirror, rewrite each saved page with its
|
||||
stylesheets, scripts, images and fonts inlined as data:
|
||||
URIs, so any page also opens on its own (links between pages
|
||||
stay relative; audio and video stay links);
|
||||
--single-file-max-size N caps each asset (default 10485760
|
||||
bytes) (--single-file)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%t</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
@@ -97,8 +97,9 @@ ${do:end-if}
|
||||
</pre>
|
||||
|
||||
${LANG_G8} :
|
||||
${/* an http: page cannot navigate to file:, so the mirror is reached through the server */}
|
||||
<a href="/website/index.html" target="_new">
|
||||
${do:output-mode:html-urlescaped}
|
||||
<a href="file://${path}/${projname}/" target="_new">
|
||||
${do:output-mode:}
|
||||
${path}/${projname}
|
||||
</a></li>
|
||||
<ul>
|
||||
|
||||
@@ -136,6 +136,13 @@ ${listid:build:LISTDEF_3}
|
||||
<tr><td><input type="checkbox" name="nopurge" ${checked:nopurge}
|
||||
title='${html:LANG_I1a}' onMouseOver="info('${html:LANG_I1a}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I57}</td></tr>
|
||||
<tr><td><input type="checkbox" name="singlefile" ${checked:singlefile}
|
||||
title='${html:LANG_SINGLEFILETIP}' onMouseOver="info('${html:LANG_SINGLEFILETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_SINGLEFILE}</td></tr>
|
||||
<tr><td>${LANG_SINGLEFILEMAX}
|
||||
<input name="singlefilemax" value="${singlefilemax}" size="12"
|
||||
title='${html:LANG_SINGLEFILEMAXTIP}' onMouseOver="info('${html:LANG_SINGLEFILEMAXTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
></td></tr>
|
||||
</table>
|
||||
|
||||
<tr><td>
|
||||
|
||||
@@ -143,6 +143,8 @@ ${do:copy:StripQuery:stripquery}
|
||||
${do:copy:StoreAllInCache:cache2}
|
||||
${do:copy:Warc:warc}
|
||||
${do:copy:WarcFile:warcfile}
|
||||
${do:copy:SingleFile:singlefile}
|
||||
${do:copy:SingleFileMaxSize:singlefilemax}
|
||||
${do:copy:LogType:logtype}
|
||||
${do:copy:UseHTTPProxyForFTP:ftpprox}
|
||||
${do:copy:ProxyType:proxytype}
|
||||
|
||||
@@ -190,6 +190,8 @@ ${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one *
|
||||
${test:cache2:--store-all-in-cache}
|
||||
${test:warc:--warc}
|
||||
${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
|
||||
${test:singlefile:--single-file}
|
||||
${test:singlefilemax:--single-file-max-size=}${unquoted:singlefilemax}
|
||||
${test:norecatch:--do-not-recatch}
|
||||
${test:logf:--single-log}
|
||||
${test:logtype:::--extra-log:--debug-log}
|
||||
@@ -242,6 +244,8 @@ StripQuery=${stripquery}
|
||||
StoreAllInCache=${ztest:cache2:0:1}
|
||||
Warc=${ztest:warc:0:1}
|
||||
WarcFile=${warcfile}
|
||||
SingleFile=${ztest:singlefile:0:1}
|
||||
SingleFileMaxSize=${singlefilemax}
|
||||
LogType=${logtype}
|
||||
UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
|
||||
ProxyType=${proxytype}
|
||||
|
||||
8
lang.def
8
lang.def
@@ -1042,3 +1042,11 @@ LANG_WARCFILE
|
||||
WARC archive name:
|
||||
LANG_WARCFILETIP
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
LANG_SINGLEFILE
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
LANG_SINGLEFILETIP
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
LANG_SINGLEFILEMAX
|
||||
Largest inlined asset (bytes):
|
||||
LANG_SINGLEFILEMAXTIP
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -1012,3 +1012,11 @@ WARC archive name:
|
||||
WARC archive name:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Largest inlined asset (bytes):
|
||||
Largest inlined asset (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -1012,3 +1012,11 @@ WARC archive name:
|
||||
Nom de l'archive WARC :
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nom de base optionnel pour l'archive WARC ; laissez vide pour le générer automatiquement dans le répertoire de sortie.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Intégrer les ressources en URI data: (pages autonomes)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Une fois le miroir terminé, réécrire chaque page enregistrée avec ses feuilles de style, scripts, images et polices intégrés, afin qu'une page puisse aussi être ouverte seule.
|
||||
Largest inlined asset (bytes):
|
||||
Taille maximale d'une ressource intégrée (octets) :
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Au-delà de cette taille, la ressource reste un lien ordinaire ; laissez vide pour la valeur par défaut de 10485760 octets.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "23 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "26 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -40,6 +40,7 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-%Z, \-\-single\-file\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
[ \fB\-KN, \-\-keep\-links[=N]\fR ]
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
@@ -198,6 +199,8 @@ delayed type check, don't make any link test but wait for files download to star
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%Z
|
||||
after the mirror, rewrite each saved page with its stylesheets, scripts, images and fonts inlined as data: URIs, so any page also opens on its own (links between pages stay relative; audio and video stay links); \-\-single\-file\-max\-size N caps each asset (default 10485760 bytes) (\-\-single\-file)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
|
||||
@@ -66,7 +66,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htswarc.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmd5.c htscodec.c htswarc.c htssinglefile.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
@@ -77,7 +77,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htsproxy.h htszlib.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htssinglefile.h htsproxy.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
htscharset.h punycode.h htsencoding.h \
|
||||
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
|
||||
|
||||
@@ -123,6 +123,10 @@ const char *hts_optalias[][4] = {
|
||||
{"warc-cdxj", "-%rc", "single", ""},
|
||||
{"wacz", "-%rz", "single",
|
||||
"package the WARC archive, CDXJ index and pages as a WACZ file"},
|
||||
{"single-file", "-%Z", "single",
|
||||
"after the mirror, inline each page's assets as data: URIs"},
|
||||
{"single-file-max-size", "-%Zs", "param1",
|
||||
"per-asset cap for --single-file, in bytes (implies it; default 10485760)"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
|
||||
@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
|
||||
/* File defs */
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -2182,6 +2183,10 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
// fin purge!
|
||||
|
||||
/* --single-file: inline each page's assets now that the tree is final (after
|
||||
the purge, so we never rewrite a file that is about to be deleted). */
|
||||
singlefile_process_mirror(opt);
|
||||
|
||||
// Indexation
|
||||
if (opt->kindex)
|
||||
index_finish(StringBuff(opt->path_html), opt->kindex);
|
||||
@@ -3643,6 +3648,10 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
to->warc_cdx = from->warc_cdx;
|
||||
to->warc_wacz = from->warc_wacz;
|
||||
|
||||
to->single_file = from->single_file;
|
||||
if (from->single_file_max_size > 0)
|
||||
to->single_file_max_size = from->single_file_max_size;
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
to->pause_max_ms = from->pause_max_ms;
|
||||
|
||||
@@ -1795,6 +1795,36 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
break;
|
||||
case 'Z': // single-file: inline each page's assets as data: URIs
|
||||
if (*(com + 1) == 's') { // --single-file-max-size N: per-asset
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option single-file-max-size needs a blank "
|
||||
"space and a size");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
{ // reject non-numeric/negative/overflow; keep the default
|
||||
char *end;
|
||||
LLint v;
|
||||
|
||||
errno = 0;
|
||||
v = strtoll(argv[na], &end, 10);
|
||||
if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
|
||||
errno != ERANGE && v > 0)
|
||||
opt->single_file_max_size = v;
|
||||
}
|
||||
opt->single_file = HTS_TRUE;
|
||||
} else {
|
||||
opt->single_file = HTS_TRUE;
|
||||
if (*(com + 1) == '0') {
|
||||
opt->single_file = HTS_FALSE;
|
||||
com++;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
|
||||
@@ -534,6 +534,11 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %Z after the mirror, rewrite each saved page with its "
|
||||
"stylesheets, scripts, images and fonts inlined as data: URIs, so "
|
||||
"any page also opens on its own (links between pages stay relative; "
|
||||
"audio and video stay links); --single-file-max-size N caps each "
|
||||
"asset (default 10485760 bytes)");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
|
||||
@@ -37,6 +37,7 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -6011,6 +6012,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->warc_file, "");
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
opt->single_file = HTS_FALSE;
|
||||
opt->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
|
||||
@@ -547,6 +547,12 @@ struct httrackp {
|
||||
archive. Tail: ABI */
|
||||
hts_boolean warc_wacz; /**< --wacz: package archive+index+pages as a WACZ zip
|
||||
(implies --warc + --warc-cdx). Tail: ABI */
|
||||
hts_boolean single_file; /**< --single-file: once the mirror is done, rewrite
|
||||
each saved page with its assets inlined as
|
||||
data: URIs. Tail: ABI */
|
||||
LLint single_file_max_size; /**< --single-file-max-size: per-asset cap in
|
||||
bytes; a bigger asset stays a link.
|
||||
Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
|
||||
@@ -59,6 +59,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscodec.h"
|
||||
#include "htsproxy.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
@@ -4708,6 +4709,297 @@ static int st_warc_wacz(httrackp *opt, int argc, char **argv) {
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* --single-file */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
static int sf_err = 0;
|
||||
|
||||
static void sf_check(int ok, const char *what) {
|
||||
if (!ok) {
|
||||
fprintf(stderr, "singlefile: %s\n", what);
|
||||
sf_err++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Write rel (a '/'-separated path under dir), creating the directories. */
|
||||
static void sf_put(const char *dir, const char *rel, const void *data,
|
||||
size_t len) {
|
||||
char BIGSTK path[HTS_URLMAXSIZE * 2];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *fp;
|
||||
|
||||
fconcat(path, sizeof(path), dir, rel);
|
||||
structcheck_utf8(path);
|
||||
fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(len == 0 || fwrite(data, 1, len, fp) == len);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Number of times needle occurs in hay. */
|
||||
static int sf_count(const char *hay, const char *needle) {
|
||||
const size_t l = strlen(needle);
|
||||
int n = 0;
|
||||
const char *p = hay;
|
||||
|
||||
while ((p = strstr(p, needle)) != NULL) {
|
||||
n++;
|
||||
p += l;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* The base64 payload following the first occurrence of prefix, up to the first
|
||||
byte outside the base64 alphabet. NULL if prefix is absent. */
|
||||
static const char *sf_payload(const char *hay, const char *prefix,
|
||||
size_t *len) {
|
||||
const char *p = strstr(hay, prefix);
|
||||
size_t n = 0;
|
||||
|
||||
if (p == NULL)
|
||||
return NULL;
|
||||
p += strlen(prefix);
|
||||
while (p[n] != '\0' && (isalnum((unsigned char) p[n]) || p[n] == '+' ||
|
||||
p[n] == '/' || p[n] == '='))
|
||||
n++;
|
||||
*len = n;
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Independent base64 decoder: the round-trip check must not lean on code64().
|
||||
*/
|
||||
static unsigned char *sf_unb64(const char *s, size_t len, size_t *outlen) {
|
||||
static const char alpha[] =
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
unsigned char *out = (unsigned char *) malloct(len / 4 * 3 + 4);
|
||||
unsigned int acc = 0;
|
||||
size_t i, n = 0;
|
||||
int bits = 0;
|
||||
|
||||
if (out == NULL)
|
||||
return NULL;
|
||||
for (i = 0; i < len; i++) {
|
||||
const char *const p = s[i] != '\0' ? strchr(alpha, s[i]) : NULL;
|
||||
|
||||
if (s[i] == '=')
|
||||
break;
|
||||
if (p == NULL) {
|
||||
freet(out);
|
||||
return NULL;
|
||||
}
|
||||
acc = (acc << 6) | (unsigned int) (p - alpha);
|
||||
bits += 6;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out[n++] = (unsigned char) ((acc >> bits) & 0xff);
|
||||
}
|
||||
}
|
||||
*outlen = n;
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Decoded payload of the first data: URI with that MIME, as a NUL-terminated
|
||||
buffer the caller freet()s. NULL when absent or undecodable. */
|
||||
static char *sf_decode(const char *hay, const char *mime, size_t *outlen) {
|
||||
char prefix[128];
|
||||
size_t len = 0, dlen = 0;
|
||||
const char *b64;
|
||||
unsigned char *raw;
|
||||
|
||||
snprintf(prefix, sizeof(prefix), "data:%s;base64,", mime);
|
||||
b64 = sf_payload(hay, prefix, &len);
|
||||
if (b64 == NULL)
|
||||
return NULL;
|
||||
raw = sf_unb64(b64, len, &dlen);
|
||||
if (raw == NULL)
|
||||
return NULL;
|
||||
raw[dlen] = '\0';
|
||||
if (outlen != NULL)
|
||||
*outlen = dlen;
|
||||
return (char *) raw;
|
||||
}
|
||||
|
||||
/* The 12-byte asset: high bytes and an embedded NUL, so a text-shaped copy
|
||||
would be caught. */
|
||||
static const char sf_png[] = "\x89PNG\r\n\x1a\n\x00\x01\x02\xff";
|
||||
#define SF_PNG_LEN 12
|
||||
|
||||
static const char sf_page[] =
|
||||
"<html><head>\n"
|
||||
"<link rel=\"stylesheet\" href=\"css/main.css\">\n"
|
||||
"<link rel=\"canonical\" href=\"other.html\">\n"
|
||||
"<title>t</title>\n"
|
||||
"<style>body { background: url(\"img/a%20b.png\"); }</style>\n"
|
||||
"</head><body>\n"
|
||||
"<img src=\"img/a%20b.png\" srcset=\"img/a%20b.png 1x, img/big.png 2x\">\n"
|
||||
"<img src=\"data:image/gif;base64,QUJD\">\n"
|
||||
"<img src=\"http://example.com/x.png\">\n"
|
||||
/* Two shapes of climbing out of the mirror, each aimed at a real image so
|
||||
a missing clamp inlines rather than merely failing to find a file: one
|
||||
resolves outside the root, one lands back inside it if a leading ".."
|
||||
is silently dropped. */
|
||||
"<img src=\"../escape.png\">\n"
|
||||
"<img src=\"../img/a%20b.png\">\n"
|
||||
"<a href=\"img/a%20b.png\">link</a>\n"
|
||||
"<video controls><source src=\"v.mp4\" type=\"video/mp4\"></video>\n"
|
||||
"<script src=\"js/app.js\"></script>\n"
|
||||
"<script>var s = \"<img src='img/a%20b.png'>\";</script>\n"
|
||||
"<div style=\"background:url(img/a%20b.png)\"></div>\n"
|
||||
"<div style='content:\"x\"; background:url(img/a%20b.png)'></div>\n"
|
||||
"</body></html>\n";
|
||||
|
||||
/* Lay a small mirror down under root and return the page path in page. */
|
||||
static void sf_fixture(const char *root, char *page, size_t pagesize) {
|
||||
static const char css[] = "@import \"sub/nested.css\";\n"
|
||||
"body { background: url(../img/a b.png); }\n"
|
||||
"/* url(../img/never.png) */\n";
|
||||
static const char nested[] = "div { background: url(../../img/a b.png); }\n";
|
||||
static const char js[] = "var app = 1;\n";
|
||||
char big[4096];
|
||||
|
||||
memset(big, 'B', sizeof(big));
|
||||
sf_put(root, "page.html", sf_page, sizeof(sf_page) - 1);
|
||||
sf_put(root, "other.html", "<html>o</html>", 14);
|
||||
sf_put(root, "css/main.css", css, sizeof(css) - 1);
|
||||
sf_put(root, "css/sub/nested.css", nested, sizeof(nested) - 1);
|
||||
sf_put(root, "js/app.js", js, sizeof(js) - 1);
|
||||
sf_put(root, "img/a b.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/big.png", big, sizeof(big));
|
||||
sf_put(root, "v.mp4",
|
||||
"\x00\x00\x00\x18"
|
||||
"ftypisom",
|
||||
12);
|
||||
fconcat(page, pagesize, root, "page.html");
|
||||
}
|
||||
|
||||
/* -#test=singlefile <dir>: rewrite a hand-built mirror and check what gets
|
||||
inlined, what must keep its link, the per-asset cap, and idempotence. */
|
||||
static int st_singlefile(httrackp *opt, int argc, char **argv) {
|
||||
char BIGSTK root[HTS_URLMAXSIZE];
|
||||
char BIGSTK page[HTS_URLMAXSIZE * 2];
|
||||
const LLint saved_cap = opt->single_file_max_size;
|
||||
char *out, *css, *nested;
|
||||
size_t outlen = 0, len = 0;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "singlefile: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
sf_err = 0;
|
||||
sf_put(argv[0], "escape.png", sf_png,
|
||||
SF_PNG_LEN); /* just outside the mirror */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror/");
|
||||
sf_fixture(root, page, sizeof(page));
|
||||
|
||||
/* Cap between the small assets and big.png. */
|
||||
opt->single_file_max_size = 1024;
|
||||
sf_check(singlefile_rewrite_file(opt, root, page),
|
||||
"first pass changed nothing");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
|
||||
/* Inlined, and the payload is the file's exact bytes. */
|
||||
{
|
||||
char *img = sf_decode(out, "image/png", &len);
|
||||
|
||||
sf_check(img != NULL && len == SF_PNG_LEN &&
|
||||
memcmp(img, sf_png, SF_PNG_LEN) == 0,
|
||||
"image payload does not round-trip");
|
||||
freet(img);
|
||||
}
|
||||
sf_check(strstr(out, "data:application/x-javascript;base64,") != NULL,
|
||||
"script not inlined");
|
||||
sf_check(strstr(out, "href=\"css/main.css\"") == NULL,
|
||||
"stylesheet not inlined");
|
||||
|
||||
/* Left alone: a page link, a navigational <link>, media, an absolute URL, an
|
||||
existing data: URI, and a reference climbing out of the mirror. */
|
||||
sf_check(strstr(out, "<a href=\"img/a%20b.png\">") != NULL, "anchor inlined");
|
||||
sf_check(strstr(out, "href=\"other.html\"") != NULL, "rel=canonical inlined");
|
||||
sf_check(strstr(out, "src=\"v.mp4\"") != NULL, "video source inlined");
|
||||
sf_check(strstr(out, "src=\"http://example.com/x.png\"") != NULL,
|
||||
"absolute URL rewritten");
|
||||
sf_check(sf_count(out, "QUJD") == 1, "existing data: URI not preserved");
|
||||
sf_check(strstr(out, "src=\"../escape.png\"") != NULL,
|
||||
"a reference outside the mirror was resolved");
|
||||
sf_check(strstr(out, "src=\"../img/a%20b.png\"") != NULL,
|
||||
"a leading .. was dropped instead of rejected");
|
||||
sf_check(strstr(out, "var s = \"<img src='img/a%20b.png'>\";") != NULL,
|
||||
"script body rewritten");
|
||||
|
||||
/* Nothing an attribute value cannot hold: url() stays unquoted, and a quote
|
||||
that was already in the CSS is escaped. */
|
||||
sf_check(strstr(out, "url(\"data:") == NULL,
|
||||
"a quoted url() would end a style attribute");
|
||||
sf_check(strstr(out, "style=\"content:"x"; background:url(data:") !=
|
||||
NULL,
|
||||
"quote inside a rewritten style attribute not escaped");
|
||||
|
||||
/* The over-cap srcset candidate keeps its link, the small one does not. */
|
||||
sf_check(strstr(out, "img/big.png 2x") != NULL, "over-cap asset inlined");
|
||||
sf_check(strstr(out, " 1x") != NULL, "srcset descriptor lost");
|
||||
sf_check(sf_count(out, "img/a%20b.png") ==
|
||||
3, /* anchor, script, the ".." one */
|
||||
"an inlinable reference was left as a link");
|
||||
|
||||
/* The inlined stylesheet carries its own @import and url() inlined. */
|
||||
css = sf_decode(out, "text/css", NULL);
|
||||
sf_check(css != NULL, "stylesheet payload undecodable");
|
||||
if (css != NULL) {
|
||||
sf_check(strstr(css, "@import \"data:text/css;base64,") != NULL,
|
||||
"@import not inlined");
|
||||
sf_check(strstr(css, "url(data:image/png;base64,") != NULL,
|
||||
"url() in stylesheet not inlined");
|
||||
sf_check(strstr(css, "url(../img/never.png)") != NULL,
|
||||
"url() inside a CSS comment was rewritten");
|
||||
nested = sf_decode(css, "text/css", NULL);
|
||||
sf_check(nested != NULL &&
|
||||
strstr(nested, "url(data:image/png;base64,") != NULL,
|
||||
"url() in the @import'ed stylesheet not inlined");
|
||||
freet(nested);
|
||||
freet(css);
|
||||
}
|
||||
|
||||
/* Idempotence: a second pass must find nothing and leave the bytes alone. */
|
||||
sf_check(!singlefile_rewrite_file(opt, root, page), "second pass rewrote");
|
||||
{
|
||||
char *again = readfile_utf8(page);
|
||||
|
||||
sf_check(again != NULL && strcmp(again, out) == 0,
|
||||
"second pass changed the page");
|
||||
freet(again);
|
||||
}
|
||||
freet(out);
|
||||
|
||||
/* Same page, a cap above big.png: it now inlines. */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror2/");
|
||||
sf_fixture(root, page, sizeof(page));
|
||||
opt->single_file_max_size = 1024 * 1024;
|
||||
sf_check(singlefile_rewrite_file(opt, root, page),
|
||||
"raised-cap pass changed nothing");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
sf_check(strstr(out, "img/big.png 2x") == NULL,
|
||||
"asset under the raised cap still a link");
|
||||
freet(out);
|
||||
|
||||
/* A cap of one byte inlines nothing at all. */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror3/");
|
||||
sf_fixture(root, page, sizeof(page));
|
||||
opt->single_file_max_size = 1;
|
||||
sf_check(!singlefile_rewrite_file(opt, root, page), "one-byte cap inlined");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
sf_check(strcmp(out, sf_page) == 0, "one-byte cap altered the page");
|
||||
freet(out);
|
||||
(void) outlen;
|
||||
|
||||
opt->single_file_max_size = saved_cap;
|
||||
printf("singlefile: %s\n", sf_err ? "FAIL" : "OK");
|
||||
return sf_err;
|
||||
}
|
||||
|
||||
// -#test=longpath <dir>: round-trip a >MAX_PATH (260) file through the file
|
||||
// wrappers, exercising hts_pathToUCS2's \\?\ prefixing on Windows (#133).
|
||||
static int st_longpath(httrackp *opt, int argc, char **argv) {
|
||||
@@ -5216,6 +5508,9 @@ static const struct selftest_entry {
|
||||
{"warc-wacz", "<dir>", "--wacz package: layout, STORE mode, sha256 digests",
|
||||
st_warc_wacz},
|
||||
#endif
|
||||
{"singlefile", "<dir>",
|
||||
"--single-file: what is inlined, the per-asset cap, idempotence",
|
||||
st_singlefile},
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
@@ -363,22 +363,6 @@ static hts_boolean body_sid_is_valid(const char *body, const char *expected) {
|
||||
return seen;
|
||||
}
|
||||
|
||||
/** Append src to the NUL-terminated dst of capacity size (NUL included).
|
||||
False, leaving dst untouched, if it would not fit: unlike strcatbuff() this
|
||||
never aborts, because every piece appended here is client-supplied. */
|
||||
static hts_boolean path_append(char *dst, size_t size, const char *src) {
|
||||
const size_t used = strlen(dst);
|
||||
const size_t len = strlen(src);
|
||||
|
||||
/* dst holds at most size-1 bytes, so "size - used" is >= 1 and the untrusted
|
||||
len stays alone: "used + len < size" could wrap and pass. */
|
||||
if (len >= size - used) {
|
||||
return HTS_FALSE;
|
||||
}
|
||||
memcpy(dst + used, src, len + 1);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Append c to dst as an HTML entity, or return HTS_FALSE if it needs none. */
|
||||
static hts_boolean cat_html_escaped(String *dst, char c) {
|
||||
switch (c) {
|
||||
@@ -427,9 +411,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
String tmpbuff = STRING_EMPTY;
|
||||
String tmpbuff2 = STRING_EMPTY;
|
||||
String fspath = STRING_EMPTY;
|
||||
/* Project directory this server set up; the only root /website/ serves from,
|
||||
and deliberately not cleared between requests. */
|
||||
String website = STRING_EMPTY;
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
|
||||
/* Load strings */
|
||||
@@ -841,11 +822,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
if (!structcheck(StringBuff(tmpbuff))) {
|
||||
FILE *fp;
|
||||
|
||||
/* Both halves of fspath come from posted fields, so a ".."
|
||||
in them would escape the mirror once served. */
|
||||
if (strstr(StringBuff(fspath), "..") == NULL) {
|
||||
StringCopy(website, StringBuff(fspath));
|
||||
}
|
||||
StringCat(tmpbuff, "winprofile.ini");
|
||||
fp = fopen(StringBuff(tmpbuff), "wb");
|
||||
if (fp != NULL) {
|
||||
@@ -929,11 +905,11 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
char *qpos;
|
||||
|
||||
/* get the URL */
|
||||
fsfile[0] = '\0';
|
||||
if (error_redirect == NULL) {
|
||||
if ((qpos = strchr(url, '?'))) {
|
||||
*qpos = '\0';
|
||||
}
|
||||
fsfile[0] = '\0';
|
||||
if (strcmp(url, "/") == 0) {
|
||||
file = "/server/index.html";
|
||||
meth = 2;
|
||||
@@ -963,23 +939,18 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
/* the override above may have swapped a mirror path for a GUI page */
|
||||
virtualpath = strncmp(file, "/website/", 9) == 0;
|
||||
|
||||
if (!virtualpath) {
|
||||
if (!path_append(fsfile, sizeof(fsfile), path) ||
|
||||
!path_append(fsfile, sizeof(fsfile), "html") ||
|
||||
!path_append(fsfile, sizeof(fsfile), file)) {
|
||||
fsfile[0] = '\0';
|
||||
}
|
||||
} else if (StringNotEmpty(website)) {
|
||||
/* Never the posted "projpath": a client root reads any file. */
|
||||
if (!path_append(fsfile, sizeof(fsfile), StringBuff(website)) ||
|
||||
!path_append(fsfile, sizeof(fsfile), "/") ||
|
||||
!path_append(fsfile, sizeof(fsfile), file + 9)) {
|
||||
fsfile[0] = '\0';
|
||||
if (strlen(path) + strlen(file) + 32 < sizeof(fsfile)) {
|
||||
if (!virtualpath) {
|
||||
sprintf(fsfile, "%shtml%s", path, file);
|
||||
} else {
|
||||
intptr_t adr = 0;
|
||||
|
||||
if (coucal_readptr(NewLangList, "projpath", &adr)) {
|
||||
sprintf(fsfile, "%s%s", (char *) adr, file + 9);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* path itself may hold ".." (webhttrack passes "<bin>/../share"), so
|
||||
only the untrusted halves are checked: file here, website above. */
|
||||
if (fsfile[0] && strstr(file, "..") == NULL
|
||||
&& (fp = fopen(fsfile, "rb"))) {
|
||||
char ok[] =
|
||||
@@ -1567,7 +1538,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
StringFree(tmpbuff);
|
||||
StringFree(tmpbuff2);
|
||||
StringFree(fspath);
|
||||
StringFree(website);
|
||||
|
||||
if (buffer)
|
||||
free(buffer);
|
||||
|
||||
1076
src/htssinglefile.c
Normal file
1076
src/htssinglefile.c
Normal file
File diff suppressed because it is too large
Load Diff
72
src/htssinglefile.h
Normal file
72
src/htssinglefile.h
Normal file
@@ -0,0 +1,72 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* --single-file: post-mirror pass embedding each page's assets as data: URIs.
|
||||
Internal, not installed. Unlike -%M (one MHT archive for the whole mirror)
|
||||
this rewrites the saved pages in place and keeps page-to-page links
|
||||
relative, so the mirror stays browsable and every page also stands alone. */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_SINGLEFILE_DEFH
|
||||
#define HTS_SINGLEFILE_DEFH
|
||||
|
||||
#include "htsopt.h"
|
||||
#include "htsstrings.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* --single-file-max-size default: a bigger asset stays an ordinary link. */
|
||||
#define SINGLEFILE_DEFAULT_MAX_SIZE (10 * 1024 * 1024)
|
||||
|
||||
/* Never base64 more than this, whatever the cap: code64() sizes in int. */
|
||||
#define SINGLEFILE_HARD_MAX_SIZE (256 * 1024 * 1024)
|
||||
|
||||
/* Rewrite every HTML page the mirror produced. No-op unless opt->single_file;
|
||||
call once the tree is final (after the update purge and the index). */
|
||||
void singlefile_process_mirror(httrackp *opt);
|
||||
|
||||
/* Rewrite one HTML document held in memory, appending the result to out.
|
||||
root is the mirror directory that references may not escape; page_path is
|
||||
the document's own path under it (both UTF-8, '/' or native separators).
|
||||
Returns HTS_TRUE if at least one reference was replaced. */
|
||||
hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
|
||||
const char *page_path, const char *html,
|
||||
size_t html_len, String *out);
|
||||
|
||||
/* singlefile_rewrite_html() over the file at page_path, rewritten in place.
|
||||
Returns HTS_TRUE if the file changed. */
|
||||
hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
|
||||
const char *page_path);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -131,6 +131,7 @@
|
||||
<ClCompile Include="htsproxy.c" />
|
||||
<ClCompile Include="htsrobots.c" />
|
||||
<ClCompile Include="htsselftest.c" />
|
||||
<ClCompile Include="htssinglefile.c" />
|
||||
<ClCompile Include="htssniff.c" />
|
||||
<ClCompile Include="htsthread.c" />
|
||||
<ClCompile Include="htstools.c" />
|
||||
|
||||
@@ -866,18 +866,12 @@ static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags)
|
||||
sprintf(headers + headersSize, "%s: "LLintP"\r\n", field, (LLint)(value)); \
|
||||
(headersSize) += (int) strlen(headers + headersSize); \
|
||||
} while(0)
|
||||
/* refvalue_size is mandatory: value comes off the cache file bounded only by
|
||||
the line buffer, which dwarfs most destinations. Clipped rather than
|
||||
rejected, since an engine-written cache carries fields wider than ours
|
||||
(contenttype is 128 there, 64 here). */
|
||||
#define ZIP_READFIELD_STRING(line, value, refline, refvalue, refvalue_size) \
|
||||
do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
(refvalue)[0] = '\0'; \
|
||||
strlncatbuff(refvalue, value, refvalue_size, (refvalue_size) - 1); \
|
||||
line[0] = '\0'; \
|
||||
} \
|
||||
} while (0)
|
||||
#define ZIP_READFIELD_STRING(line, value, refline, refvalue) do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
strcpy(refvalue, value); \
|
||||
line[0] = '\0'; \
|
||||
} \
|
||||
} while(0)
|
||||
#define ZIP_READFIELD_INT(line, value, refline, refvalue) do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
int intval = 0; \
|
||||
@@ -1080,23 +1074,16 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
value++;
|
||||
ZIP_READFIELD_INT(line, value, "X-In-Cache", dataincache);
|
||||
ZIP_READFIELD_INT(line, value, "X-Statuscode", r->statuscode);
|
||||
ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg,
|
||||
sizeof(r->msg));
|
||||
ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg); // msg
|
||||
ZIP_READFIELD_INT(line, value, "X-Size", r->size); // size
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype,
|
||||
sizeof(r->contenttype));
|
||||
ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset,
|
||||
sizeof(r->charset));
|
||||
ZIP_READFIELD_STRING(line, value, "Last-Modified",
|
||||
r->lastmodified, sizeof(r->lastmodified));
|
||||
ZIP_READFIELD_STRING(line, value, "Etag", r->etag,
|
||||
sizeof(r->etag));
|
||||
ZIP_READFIELD_STRING(line, value, "Location", r->location,
|
||||
sizeof(location_default));
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype
|
||||
ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset); // contenttype
|
||||
ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified
|
||||
ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag
|
||||
ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition",
|
||||
r->cdispo, sizeof(r->cdispo));
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
|
||||
sizeof(previous_save_));
|
||||
r->cdispo); // Content-disposition
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename
|
||||
if (line[0] != '\0') {
|
||||
int len = r->headers ? ((int) strlen(r->headers)) : 0;
|
||||
int nlen =
|
||||
|
||||
303
tests/82_local-single-file.test
Normal file
303
tests/82_local-single-file.test
Normal file
@@ -0,0 +1,303 @@
|
||||
#!/bin/bash
|
||||
# Issue #713: --single-file rewrites the mirrored pages so their assets ride
|
||||
# along as data: URIs. End to end over a real crawl, with the payloads decoded
|
||||
# and compared against what the server actually served.
|
||||
set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_713.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
htssrv=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
|
||||
test -z "$htssrv" || kill -9 "$htssrv" 2>/dev/null || true
|
||||
wait "$htssrv" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
htssrv=
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# --- the site ---------------------------------------------------------------
|
||||
doc="${tmpdir}/doc"
|
||||
mkdir -p "$doc"
|
||||
cat >"${doc}/index.html" <<'EOF'
|
||||
<html><head>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<link rel="canonical" href="other.html">
|
||||
</head><body>
|
||||
<img src="pixel.svg" alt="p">
|
||||
<img src="big.svg" alt="b">
|
||||
<script src="app.js"></script>
|
||||
<a href="other.html">other</a>
|
||||
</body></html>
|
||||
EOF
|
||||
cat >"${doc}/other.html" <<'EOF'
|
||||
<html><body><a href="index.html">back</a></body></html>
|
||||
EOF
|
||||
cat >"${doc}/style.css" <<'EOF'
|
||||
@import "nested.css";
|
||||
body { background: url(pixel.svg); }
|
||||
EOF
|
||||
cat >"${doc}/nested.css" <<'EOF'
|
||||
div { background: url(pixel.svg); }
|
||||
EOF
|
||||
printf 'var app = 1;\n' >"${doc}/app.js"
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><rect width="1" height="1"/></svg>\n' \
|
||||
>"${doc}/pixel.svg"
|
||||
{
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><!--'
|
||||
head -c 20000 /dev/zero | tr '\0' 'x'
|
||||
printf '%s\n' '--><rect width="1" height="1"/></svg>'
|
||||
} >"${doc}/big.svg"
|
||||
|
||||
# --- server -----------------------------------------------------------------
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
"$python" "$server" --root "$(nativepath "$doc")" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- the rewriter in isolation, over a hand-built mirror --------------------
|
||||
printf '[engine self-test] ..\t'
|
||||
st=$(httrack -O /dev/null "-#test=singlefile" "${tmpdir}/st/" 2>/dev/null)
|
||||
case "$st" in
|
||||
*": OK") echo OK ;;
|
||||
*)
|
||||
echo "FAIL: $st"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- crawl ------------------------------------------------------------------
|
||||
# The cap sits between pixel.svg and big.svg, so one inlines and one must not.
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=0)
|
||||
httrack "${common[@]}" --single-file --single-file-max-size 4096 \
|
||||
"${base}/index.html" >"${tmpdir}/log1" 2>&1
|
||||
|
||||
page="${out}/127.0.0.1_${port}/index.html"
|
||||
test -f "$page" || {
|
||||
echo "FAIL: no mirrored page at $page"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- what must be inlined, byte for byte ------------------------------------
|
||||
# The decoder is python, not httrack, so a broken encoder cannot agree with
|
||||
# itself here.
|
||||
extract() {
|
||||
"$python" - "$1" "$2" "$3" <<'PY'
|
||||
import base64, re, sys
|
||||
html = open(sys.argv[1], "rb").read().decode("utf-8", "replace")
|
||||
m = re.search(r'data:%s;base64,([A-Za-z0-9+/=]*)' % re.escape(sys.argv[2]), html)
|
||||
if m is None:
|
||||
sys.exit(1)
|
||||
open(sys.argv[3], "wb").write(base64.b64decode(m.group(1)))
|
||||
PY
|
||||
}
|
||||
|
||||
printf '[image inlined and identical] ..\t'
|
||||
extract "$page" "image/svg+xml" "${tmpdir}/got.svg" || {
|
||||
echo "FAIL: no inlined image"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/pixel.svg" "${tmpdir}/got.svg" || {
|
||||
echo "FAIL: inlined image differs from the served file"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[script inlined and identical] ..\t'
|
||||
extract "$page" "application/x-javascript" "${tmpdir}/got.js" || {
|
||||
echo "FAIL: no inlined script"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/app.js" "${tmpdir}/got.js" || {
|
||||
echo "FAIL: inlined script differs from the served file"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[stylesheet inlined, recursively] ..\t'
|
||||
extract "$page" "text/css" "${tmpdir}/got.css" || {
|
||||
echo "FAIL: no inlined stylesheet"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'url(data:image/svg+xml;base64,' "${tmpdir}/got.css" || {
|
||||
echo "FAIL: url() inside the inlined stylesheet was not inlined"
|
||||
exit 1
|
||||
}
|
||||
extract "${tmpdir}/got.css" "text/css" "${tmpdir}/got2.css" || {
|
||||
echo "FAIL: @import was not inlined"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'url(data:image/svg+xml;base64,' "${tmpdir}/got2.css" || {
|
||||
echo "FAIL: url() inside the @import'ed stylesheet was not inlined"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- what must keep its link ------------------------------------------------
|
||||
printf '[page links stay relative] ..\t'
|
||||
grep -q 'href="other.html"' "$page" || {
|
||||
echo "FAIL: the link to another page was not left relative"
|
||||
exit 1
|
||||
}
|
||||
test -f "${out}/127.0.0.1_${port}/other.html" || {
|
||||
echo "FAIL: the linked page is missing from the mirror"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[over-cap asset stays a link] ..\t'
|
||||
grep -q 'src="big.svg"' "$page" || {
|
||||
echo "FAIL: the over-cap asset did not keep its link"
|
||||
exit 1
|
||||
}
|
||||
test -f "${out}/127.0.0.1_${port}/big.svg" || {
|
||||
echo "FAIL: the over-cap asset is missing from the mirror"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'over the 4096-byte cap' "${out}/hts-log.txt" || {
|
||||
echo "FAIL: the over-cap asset was not reported in the log"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[the mirror still works] ..\t'
|
||||
if grep -q 'href="style.css"' "$page"; then
|
||||
echo "FAIL: the stylesheet link survived"
|
||||
exit 1
|
||||
fi
|
||||
test -f "${out}/127.0.0.1_${port}/style.css" || {
|
||||
echo "FAIL: the mirror lost the standalone stylesheet"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- a second run must not re-encode ----------------------------------------
|
||||
printf '[idempotent under --update] ..\t'
|
||||
# HTTrack stamps the time into its own footer comment, so compare without it.
|
||||
grep -v 'Mirrored from' "$page" >"${tmpdir}/page1.html"
|
||||
httrack "${common[@]}" --single-file --single-file-max-size 4096 --update \
|
||||
"${base}/index.html" >"${tmpdir}/log2" 2>&1
|
||||
grep -v 'Mirrored from' "$page" >"${tmpdir}/page2.html"
|
||||
cmp -s "${tmpdir}/page1.html" "${tmpdir}/page2.html" || {
|
||||
echo "FAIL: the second run changed the page"
|
||||
exit 1
|
||||
}
|
||||
# A double-encoded payload would decode to another data: URI, not to the SVG.
|
||||
extract "$page" "image/svg+xml" "${tmpdir}/got2.svg" || {
|
||||
echo "FAIL: no inlined image after the second run"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/pixel.svg" "${tmpdir}/got2.svg" || {
|
||||
echo "FAIL: the second run re-encoded the inlined image"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- the web GUI must emit the same options ---------------------------------
|
||||
printf '[webhttrack renders the options] ..\t'
|
||||
command -v htsserver >/dev/null || {
|
||||
echo "no htsserver in PATH"
|
||||
exit 1
|
||||
}
|
||||
distdir=$(cd "${top_srcdir}" && pwd)
|
||||
sport=$("$python" -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()')
|
||||
srvlog="${tmpdir}/htsserver.log"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
export HOME="${tmpdir}/home"
|
||||
mkdir -p "$HOME"
|
||||
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
|
||||
) &
|
||||
htssrv=$!
|
||||
url=
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "$srvlog") && test -n "$url" && break
|
||||
kill -0 "$htssrv" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || {
|
||||
echo "FAIL: htsserver did not start: $(cat "$srvlog")"
|
||||
exit 1
|
||||
}
|
||||
"$python" - "$url" <<'PY' || exit 1
|
||||
import re, sys, urllib.parse, urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
form = urllib.request.urlopen(url + "server/index.html", timeout=20).read()
|
||||
m = re.search(rb'name="sid" value="([0-9a-f]+)"', form)
|
||||
if not m:
|
||||
sys.exit("FAIL: no session id in server/index.html")
|
||||
fields = [("sid", m.group(1).decode()), ("singlefile", "on"),
|
||||
("singlefilemax", "5000")]
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v)) for k, v in fields)
|
||||
req = urllib.request.Request(url + "server/step4.html",
|
||||
data=body.encode("latin-1"), method="POST")
|
||||
page = urllib.request.urlopen(req, timeout=20).read().decode("latin-1")
|
||||
|
||||
|
||||
def textarea(name):
|
||||
m = re.search(r'<textarea name="%s".*?>(.*?)</textarea>' % name, page, re.S)
|
||||
if m is None:
|
||||
sys.exit("FAIL: no %s textarea in the rendered step4.html" % name)
|
||||
return m.group(1)
|
||||
|
||||
|
||||
cmd = textarea("command")
|
||||
for want in ("--single-file", "--single-file-max-size=5000"):
|
||||
if want not in cmd:
|
||||
sys.exit("FAIL: %s missing from the command line\n%s" % (want, cmd))
|
||||
ini = textarea("winprofile").replace("\r\n", "\n")
|
||||
for want in ("\nSingleFile=1\n", "\nSingleFileMaxSize=5000\n"):
|
||||
if want not in ini:
|
||||
sys.exit("FAIL: %r missing from winprofile.ini\n%s" % (want, ini))
|
||||
PY
|
||||
echo OK
|
||||
|
||||
printf '[project reload maps the keys back] ..\t'
|
||||
grep -q 'do:copy:SingleFile:singlefile' "${distdir}/html/server/step2.html" || {
|
||||
echo "FAIL: step2.html does not restore SingleFile"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'do:copy:SingleFileMaxSize:singlefilemax' "${distdir}/html/server/step2.html" || {
|
||||
echo "FAIL: step2.html does not restore SingleFileMaxSize"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
@@ -1,134 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A browser will not follow an http: page to a file: URL, so the GUI's "browse
|
||||
# the mirror" link has to go through the server's own /website/ route.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
python=$(find_python) || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
# Control: on a path that does not resolve, grep "passes" without reading anything.
|
||||
test -f "${distdir}/html/server/finished.html" || fail "no GUI pages under ${distdir}"
|
||||
bad=$(grep -l 'file://' "${distdir}"/html/server/finished.html || true)
|
||||
test -z "${bad}" || fail "file: link left in: ${bad}"
|
||||
|
||||
work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_browse.XXXXXX") || fail "no tmpdir"
|
||||
srvlog=$(mktemp)
|
||||
srv=
|
||||
srvpid=
|
||||
cleanup() {
|
||||
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
|
||||
test -z "${srvpid}" || kill -9 "${srvpid}" 2>/dev/null || true
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
rm -rf "${work}" "${srvlog}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# Stand in for a finished mirror: the crawl itself is not under test.
|
||||
proj="${work}/websites/proj"
|
||||
mkdir -p "${proj}"
|
||||
printf 'MIRROR-PROBE-OK\n' >"${proj}/probe.txt"
|
||||
printf '<html><body>MIRROR-INDEX-OK</body></html>\n' >"${proj}/index.html"
|
||||
|
||||
# An isolated HOME keeps a stray ~/.httrack.ini out of the server's settings.
|
||||
sport=$("${python}" -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()')
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
export HOME="${work}"
|
||||
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
|
||||
) &
|
||||
srv=$!
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
|
||||
kill -0 "${srv}" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
|
||||
srvpid=$(sed -n 's/^PID=//p' "${srvlog}") # absent on Windows
|
||||
|
||||
# htsserver resolves the posted paths itself, so hand it native ones.
|
||||
"${python}" - "${url}" "$(nativepath "${work}/websites")" "$(nativepath "${proj}")" <<'PY' || fail "browse-link checks failed"
|
||||
import re, sys, urllib.error, urllib.parse, urllib.request
|
||||
|
||||
url, base, proj = sys.argv[1].rstrip("/"), sys.argv[2], sys.argv[3]
|
||||
rc = 0
|
||||
|
||||
|
||||
def check(ok, what):
|
||||
global rc
|
||||
print(("ok: " if ok else "FAIL: ") + what)
|
||||
if not ok:
|
||||
rc = 1
|
||||
|
||||
|
||||
def get(path):
|
||||
# The UI is served ISO-8859-1, so stay in bytes.
|
||||
return urllib.request.urlopen(url + path, timeout=20).read()
|
||||
|
||||
|
||||
# No crawl has run, so no commandRunning/commandEnd override swaps the page out.
|
||||
finished = get("/server/finished.html").decode("latin-1")
|
||||
# Control: an empty or unrendered page would pass "no file:" on its own.
|
||||
check("HTTrack Website Copier" in finished, "finished.html rendered")
|
||||
check("file://" not in finished, "finished.html has no file: link")
|
||||
|
||||
# Control: /website/ 404s until a project is set, so the fetches below are real.
|
||||
try:
|
||||
get("/website/probe.txt")
|
||||
check(False, "/website/ served with no project set")
|
||||
except urllib.error.HTTPError as e:
|
||||
check(e.code == 404, "/website/ is bound to a project (got %d)" % e.code)
|
||||
|
||||
# Point the server at the mirror the way step4.html does; a body without the
|
||||
# session id is refused outright.
|
||||
sid = re.search(r'name="sid" value="([0-9a-f]+)"', finished)
|
||||
if sid is None:
|
||||
print("FAIL: no sid in the rendered form")
|
||||
sys.exit(1)
|
||||
fields = [("sid", sid.group(1)), ("path", base), ("projname", "proj"),
|
||||
("projpath", proj + "/")]
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="")) for k, v in fields)
|
||||
urllib.request.urlopen(urllib.request.Request(
|
||||
url + "/server/step4.html", data=body.encode("latin-1"), method="POST"),
|
||||
timeout=20).read()
|
||||
|
||||
check(get("/website/probe.txt") == b"MIRROR-PROBE-OK\n", "mirror file over http")
|
||||
check("MIRROR-INDEX-OK" in get("/website/index.html").decode("latin-1"),
|
||||
"mirror index over http")
|
||||
|
||||
body = get("/server/finished.html").decode("latin-1")
|
||||
# Match the anchor by its mirror-path label: the list below it links /website/
|
||||
# too, so a bare "is the route mentioned" check passes on the unfixed page.
|
||||
check(re.search(r'href="/website/index\.html"[^>]*>\s*' +
|
||||
re.escape(base + "/proj"), body) is not None,
|
||||
"the mirror-path link points at the served mirror")
|
||||
check("file://" not in body, "finished.html has no file: link")
|
||||
|
||||
sys.exit(rc)
|
||||
PY
|
||||
|
||||
# A leaked htsserver wedges the parallel harness behind a green log.
|
||||
cleanup
|
||||
! kill -0 "${srv}" 2>/dev/null || fail "htsserver ${srv} survived"
|
||||
|
||||
echo "PASS"
|
||||
@@ -44,14 +44,15 @@ print(s.getsockname()[1])
|
||||
s.close()'
|
||||
}
|
||||
|
||||
# Echo the announced URL.
|
||||
# Echo the announced URL. Extra args become key/value pairs in the server store,
|
||||
# which is how the project path gets set here; the GUI posts it from step4.
|
||||
start() {
|
||||
local port url
|
||||
port=$(freeport)
|
||||
: >"${log}"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
|
||||
exec htsserver "${distdir}/" --port "${port}" "$@" >"${log}" 2>&1
|
||||
) &
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
|
||||
@@ -100,15 +101,19 @@ s.close()' "${port}" "$@" >/dev/null
|
||||
|
||||
# LF-only, and a line ending in a backslash: the expander rewrites both, so a
|
||||
# mangled reply fails the byte comparison even where no directive is present.
|
||||
mirror="${work}/proj/hostile.html"
|
||||
mirror="${work}/hostile.html"
|
||||
# shellcheck disable=SC2016 # the directives are the payload, not shell expansions
|
||||
printf 'Hostile mirrored page.\nsid=${_sid} copy=${sid}\ntrailing backslash: \\\n' \
|
||||
>"${mirror}"
|
||||
|
||||
hdr="${work}/hdr"
|
||||
body="${work}/body"
|
||||
|
||||
# The server merges $HOME/.httrack.ini into the same store on the first request;
|
||||
# point it somewhere empty so a developer's own file cannot shadow the fields.
|
||||
# point it somewhere empty so a developer's own file cannot shadow projpath.
|
||||
export HOME="${work}"
|
||||
|
||||
url=$(start)
|
||||
url=$(start projpath "${work}/")
|
||||
port=$(portof "${url}")
|
||||
|
||||
# Positive control: the GUI's own templates must still expand. This is also
|
||||
@@ -117,17 +122,6 @@ fetch "${port}" /server/index.html "${hdr}" "${body}"
|
||||
sid=$(sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' "${body}" | head -1)
|
||||
test "${#sid}" -eq 32 || fail "GUI page did not expand \${sid} (got '${sid}')"
|
||||
|
||||
# /website/ serves only the root the server itself recorded, so save a profile
|
||||
# (no command_do=start, so nothing crawls) to create it, then plant the file.
|
||||
post "${port}" "sid=${sid}" command=httrack command_do=save winprofile=x \
|
||||
"path=${work}" projname=proj
|
||||
test -f "${work}/proj/hts-cache/winprofile.ini" ||
|
||||
fail "profile save did not create the project: $(cat "${log}")"
|
||||
mkdir -p "$(dirname "${mirror}")"
|
||||
# shellcheck disable=SC2016 # the directives are the payload, not shell expansions
|
||||
printf 'Hostile mirrored page.\nsid=${_sid} copy=${sid}\ntrailing backslash: \\\n' \
|
||||
>"${mirror}"
|
||||
|
||||
fetch "${port}" /website/hostile.html "${hdr}" "${body}"
|
||||
grep -q '^HTTP/1\.0 200 ' "${hdr}" || fail "mirrored page not served: $(head -1 "${hdr}")"
|
||||
grep -qF "${sid}" "${body}" &&
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# /website/ is served from the project directory htsserver set up itself, never
|
||||
# from a root the request body names, and composing that path must stay bounded.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
command -v python3 >/dev/null || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
srv=
|
||||
log=$(mktemp)
|
||||
base=$(mktemp -d)
|
||||
cleanup() {
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
rm -f "${log}"
|
||||
rm -rf "${base}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()'
|
||||
}
|
||||
|
||||
# Echo the announced URL. Runs in a command substitution, so it is a
|
||||
# subshell and cannot export the pid: the caller reads it back with srvpid.
|
||||
start() {
|
||||
local port url
|
||||
port=$(freeport)
|
||||
: >"${log}"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
|
||||
) &
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
|
||||
echo "${url}"
|
||||
}
|
||||
|
||||
# The server reports its own pid; the aliveness assertion below hangs off it.
|
||||
srvpid() { sed -n 's/^PID=//p' "${log}" | head -1; }
|
||||
|
||||
alive() { kill -0 "$1" 2>/dev/null; }
|
||||
|
||||
portof() { echo "${1##*:}" | tr -d /; }
|
||||
|
||||
# Raw request to 127.0.0.1:$1: GET the path $2, or POST the body $3 to / when
|
||||
# $2 is empty. Prints the reply.
|
||||
request() {
|
||||
python3 -c 'import socket, sys
|
||||
port, path, body = int(sys.argv[1]), sys.argv[2], sys.argv[3]
|
||||
if path:
|
||||
req = "GET %s HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % path
|
||||
else:
|
||||
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
|
||||
"Content-type: application/x-www-form-urlencoded\r\n"
|
||||
"Content-length: %d\r\n\r\n%s" % (len(body), body))
|
||||
s = socket.create_connection(("127.0.0.1", port), 10)
|
||||
s.settimeout(30)
|
||||
s.sendall(req.encode())
|
||||
out = b""
|
||||
while True:
|
||||
b = s.recv(65536)
|
||||
if not b:
|
||||
break
|
||||
out += b
|
||||
s.close()
|
||||
sys.stdout.write(out.decode("latin-1"))' "$1" "$2" "$3"
|
||||
}
|
||||
|
||||
get() { request "$1" "$2" ""; }
|
||||
post() { request "$1" "" "$2"; }
|
||||
|
||||
url=$(start)
|
||||
port=$(portof "${url}")
|
||||
srv=$(srvpid)
|
||||
test -n "${srv}" || fail "htsserver did not report its pid"
|
||||
|
||||
# Every request body is gated by the session id (78_webhttrack-sid.test).
|
||||
sid=$(get "${port}" /server/index.html |
|
||||
sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1)
|
||||
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid (got '${sid}')"
|
||||
|
||||
# A file the mirror must never expose, next to the project that may.
|
||||
echo "SECRETMARKER" >"${base}/secret.txt"
|
||||
mkdir -p "${base}/proj"
|
||||
echo "LOGMARKER" >"${base}/proj/hts-log.txt"
|
||||
|
||||
# error_redirect is the branch that skipped the fsfile clearing, so fail the save
|
||||
# with a component over NAME_MAX (mkdir refuses it whatever the uid). Must precede
|
||||
# any successful save: commandEnd then swaps the error page for the finished one.
|
||||
get "${port}" /server/style.css >/dev/null # leaves a path behind in fsfile
|
||||
post "${port}" "sid=${sid}&command=httrack&command_do=save&winprofile=x&path=${base}/$(printf '%0300d' 0)&projname=p" |
|
||||
grep -q '^Location: /server/error.html' ||
|
||||
fail "the refused save did not redirect to the error page"
|
||||
|
||||
# No project yet, so no root to serve from.
|
||||
post "${port}" "sid=${sid}&projpath=${base}/" >/dev/null
|
||||
get "${port}" /website/secret.txt | grep -q SECRETMARKER &&
|
||||
fail "a posted projpath served a file outside any project"
|
||||
|
||||
# Positive control: step4's "save settings" flow registers the project without
|
||||
# crawling, and browsing its mirror keeps working. Without it the assertions
|
||||
# around it would pass on a server that never serves /website/ at all.
|
||||
body="sid=${sid}&command=httrack&command_do=save&winprofile=x"
|
||||
body="${body}&path=${base}&projname=proj&projpath=${base}/proj/"
|
||||
post "${port}" "${body}" >/dev/null
|
||||
test -f "${base}/proj/hts-cache/winprofile.ini" ||
|
||||
fail "the project was not registered: $(cat "${log}")"
|
||||
get "${port}" /website/hts-log.txt | grep -q LOGMARKER ||
|
||||
fail "the registered project's mirror is not served"
|
||||
|
||||
# Same request with the root repointed: the project is legitimate, projpath is
|
||||
# not what decides where the bytes come from.
|
||||
post "${port}" "sid=${sid}&projpath=${base}/" >/dev/null
|
||||
get "${port}" /website/secret.txt | grep -q SECRETMARKER &&
|
||||
fail "a posted projpath repointed the served root"
|
||||
post "${port}" "sid=${sid}&projpath=/etc/" >/dev/null
|
||||
get "${port}" /website/passwd | grep -q '^root:' &&
|
||||
fail "a posted projpath read an arbitrary system file"
|
||||
|
||||
# A ".." anywhere in the recorded root would escape the mirror on every later
|
||||
# request, so the save must be refused and the previous root kept.
|
||||
post "${port}" "sid=${sid}&command=httrack&command_do=save&winprofile=x&path=${base}/proj&projname=.." >/dev/null
|
||||
get "${port}" /website/secret.txt | grep -q SECRETMARKER &&
|
||||
fail "a '..' in the saved project path escaped the mirror root"
|
||||
get "${port}" /website/hts-log.txt | grep -q LOGMARKER ||
|
||||
fail "rejecting the '..' root also lost the previous one"
|
||||
|
||||
# The root is now the server's own, but it is still built from two posted
|
||||
# fields: 800-odd bytes of them used to be sprintf'd into a 1024-byte buffer.
|
||||
seg=$(printf '%0200d' 0)
|
||||
longpath="${base}/${seg}/${seg}/${seg}/${seg}"
|
||||
fspath="${longpath}/proj"
|
||||
# structcheck() refuses a root over HTS_URLMAXSIZE, so the URL carries the rest.
|
||||
test "$((${#fspath} + 11))" -le 1024 ||
|
||||
fail "the long project path (${#fspath}) would not pass structcheck"
|
||||
longurl=$(printf '%0800d' 0)
|
||||
test "$((${#fspath} + 1 + ${#longurl}))" -gt 1024 ||
|
||||
fail "the composed path (${#fspath} + ${#longurl}) would not overflow"
|
||||
post "${port}" "sid=${sid}&command=httrack&command_do=save&winprofile=x&path=${longpath}&projname=proj" >/dev/null
|
||||
test -f "${fspath}/hts-cache/winprofile.ini" ||
|
||||
fail "the long-path project was not registered: $(cat "${log}")"
|
||||
get "${port}" "/website/${longurl}" >/dev/null 2>&1 || true
|
||||
alive "${srv}" || fail "an over-long project path crashed the server: $(cat "${log}")"
|
||||
get "${port}" /server/index.html | grep -q '200 OK' ||
|
||||
fail "the server stopped answering after the over-long project path"
|
||||
|
||||
echo "PASS"
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/bin/bash
|
||||
# A cached header longer than proxytrack's field must be clipped, not copied
|
||||
# raw: contenttype[64] sits just before the location pointer, so an unbounded
|
||||
# copy smashed it and the next Location line wrote through the wreckage.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
python=$(find_python) || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# proxytrack reads its metadata from each entry's ZIP local extra field
|
||||
"$python" - "$dir/in.zip" <<'EOF'
|
||||
import sys, zipfile
|
||||
|
||||
body = b"hello world"
|
||||
ct = "text/html" + "A" * 400
|
||||
meta = (
|
||||
"X-In-Cache: 1\r\n"
|
||||
"X-StatusCode: 200\r\n"
|
||||
"X-StatusMessage: OK\r\n"
|
||||
"X-Size: %d\r\n"
|
||||
"Content-Type: %s\r\n"
|
||||
"X-Charset: iso-8859-1\r\n"
|
||||
# a parseable date is required: the ARC writer dereferences it unchecked
|
||||
"Last-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\n"
|
||||
"Location: http://example.com/%s\r\n"
|
||||
"X-Save: out/page.html\r\n" % (len(body), ct, "B" * 400)
|
||||
).encode()
|
||||
|
||||
zi = zipfile.ZipInfo("example.com/page.html")
|
||||
zi.compress_type = zipfile.ZIP_STORED
|
||||
zi.extra = meta
|
||||
with zipfile.ZipFile(sys.argv[1], "w") as z:
|
||||
z.writestr(zi, body)
|
||||
EOF
|
||||
|
||||
proxytrack --convert "$dir/out.arc" "$dir/in.zip" >/dev/null 2>&1 || {
|
||||
echo "FAIL: proxytrack crashed on a cache with over-long header fields" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
test -s "$dir/out.arc" || {
|
||||
echo "FAIL: entry dropped instead of clipped" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# clipped to contenttype[64], so 63 characters survive: "text/html" plus 54 A's
|
||||
longest=$(grep -ao 'A\+' "$dir/out.arc" | awk '{ print length($0) }' | sort -rn | head -n1)
|
||||
test "$longest" = 54 || {
|
||||
echo "FAIL: expected the content-type clipped to 63 chars (54 A's), got ${longest:-none}" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -179,10 +179,8 @@ TESTS = \
|
||||
79_local-proxytrack-webdav-mime.test \
|
||||
80_engine-crash-symbolize.test \
|
||||
81_webhttrack-maxsize.test \
|
||||
82_webhttrack-browse-links.test \
|
||||
82_local-single-file.test \
|
||||
83_webhttrack-argescape.test \
|
||||
84_webhttrack-mirror-verbatim.test \
|
||||
85_webhttrack-projpath.test \
|
||||
86_local-proxytrack-cache-longfields.test
|
||||
84_webhttrack-mirror-verbatim.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -46,11 +46,15 @@ cat >"$stubdir/x-www-browser" <<EOF
|
||||
echo "stub browser invoked with: \$1" >&2
|
||||
# Also fetch an option page and require a rendered title='' tooltip: proves the
|
||||
# option template expands and the \${html:} filter escapes into the attribute.
|
||||
# option9 additionally proves the WARC control renders with its expanded label.
|
||||
# option9 additionally proves the WARC control renders with its expanded label,
|
||||
# and option2 the --single-file pair: on field names plus the absence of an
|
||||
# unexpanded key, since the default locale here is French.
|
||||
opturl="\${1%/}/server/option2.html"
|
||||
warcurl="\${1%/}/server/option9.html"
|
||||
if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html &&
|
||||
opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='" &&
|
||||
printf '%s' "\$opt" | grep -qaF 'name="singlefile"' && printf '%s' "\$opt" | grep -qaF 'name="singlefilemax"' &&
|
||||
! printf '%s' "\$opt" | grep -qaF '\${LANG_SINGLEFILE}' &&
|
||||
warc="\$(curl -fsSL --max-time 20 "\$warcurl")" && printf '%s' "\$warc" | grep -qaF 'name="warcfile"' && printf '%s' "\$warc" | grep -qaF WARC; then
|
||||
echo PASS >"$marker"
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user