mirror of
https://github.com/xroche/httrack.git
synced 2026-07-27 11:03:13 +03:00
Compare commits
19 Commits
cleanup/de
...
feat/singl
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0130473dc1 | ||
|
|
8b72e3dddf | ||
|
|
5ee188ba1a | ||
|
|
d3a915641f | ||
|
|
15e2c701a5 | ||
|
|
6bd4209c94 | ||
|
|
a228cede8a | ||
|
|
b5db9bb7ed | ||
|
|
37ee129810 | ||
|
|
06dc54fa44 | ||
|
|
42072f54b0 | ||
|
|
5f3787eb28 | ||
|
|
4a62cfb75b | ||
|
|
906906efa7 | ||
|
|
f50307d878 | ||
|
|
c6b7c407ed | ||
|
|
a1226401e0 | ||
|
|
71ca31e2ec | ||
|
|
b90d699d2e |
5
.github/workflows/windows-build.yml
vendored
5
.github/workflows/windows-build.yml
vendored
@@ -227,8 +227,9 @@ jobs:
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
|
||||
# webdav-mime needs a reapable background listener, which MSYS cannot give it;
|
||||
# badmtime needs a filesystem that stores an mtime past gmtime's range.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test"
|
||||
# badmtime needs a filesystem that stores an mtime past gmtime's range;
|
||||
# single-file ends on a GUI half needing htsserver, which this job does not build.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
if FUZZERS
|
||||
noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
|
||||
fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx \
|
||||
fuzz-htsparse
|
||||
fuzz-htsparse fuzz-singlefile
|
||||
endif
|
||||
|
||||
AM_CPPFLAGS = \
|
||||
@@ -27,6 +27,7 @@ fuzz_url_SOURCES = fuzz-url.c fuzz.h
|
||||
fuzz_header_SOURCES = fuzz-header.c fuzz.h
|
||||
fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
|
||||
fuzz_htsparse_SOURCES = fuzz-htsparse.c fuzz.h
|
||||
fuzz_singlefile_SOURCES = fuzz-singlefile.c fuzz.h
|
||||
|
||||
# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
|
||||
EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
@@ -47,4 +48,8 @@ EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
corpus/cachendx/regress-overadvance.bin \
|
||||
corpus/cachendx/regress-truncated-entry.bin \
|
||||
corpus/htsparse/basic.html corpus/htsparse/script-inscript.html \
|
||||
corpus/htsparse/meta-usemap.html corpus/htsparse/malformed.html
|
||||
corpus/htsparse/meta-usemap.html corpus/htsparse/malformed.html \
|
||||
corpus/singlefile/img-src.html corpus/singlefile/link-rel.html \
|
||||
corpus/singlefile/style-block.html corpus/singlefile/style-attr.html \
|
||||
corpus/singlefile/srcset.html corpus/singlefile/rawtext.html \
|
||||
corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html
|
||||
|
||||
3
fuzz/corpus/singlefile/img-src.html
Normal file
3
fuzz/corpus/singlefile/img-src.html
Normal file
@@ -0,0 +1,3 @@
|
||||
<img src="a.png">
|
||||
<img src=big.png alt=over-cap>
|
||||
<img src="../escape.png"><img src="/abs.png"><img src="data:,x">
|
||||
4
fuzz/corpus/singlefile/link-rel.html
Normal file
4
fuzz/corpus/singlefile/link-rel.html
Normal file
@@ -0,0 +1,4 @@
|
||||
<link rel="stylesheet" href="s.css">
|
||||
<link rel=icon href=a.png>
|
||||
<link rel="next" href="p2.html">
|
||||
<link rel="preload" href="j.js">
|
||||
5
fuzz/corpus/singlefile/malformed.html
Normal file
5
fuzz/corpus/singlefile/malformed.html
Normal file
@@ -0,0 +1,5 @@
|
||||
<img src="unterminated.png
|
||||
<div style="background:url(a.png">
|
||||
<style>@import url(
|
||||
<!-- unterminated comment
|
||||
<a href=
|
||||
1
fuzz/corpus/singlefile/many-attrs.html
Normal file
1
fuzz/corpus/singlefile/many-attrs.html
Normal file
@@ -0,0 +1 @@
|
||||
<img src="a.png" a0="v" a1="v" a2="v" a3="v" a4="v" a5="v" a6="v" a7="v" a8="v" a9="v" a10="v" a11="v" a12="v" a13="v" a14="v" a15="v" a16="v" a17="v" a18="v" a19="v" a20="v" a21="v" a22="v" a23="v" a24="v" a25="v" a26="v" a27="v" a28="v" a29="v" a30="v" a31="v" a32="v" a33="v" a34="v" a35="v" a36="v" a37="v" a38="v" a39="v" a40="v" a41="v" a42="v" a43="v" a44="v" a45="v" a46="v" a47="v" a48="v" a49="v" a50="v" a51="v" a52="v" a53="v" a54="v" a55="v" a56="v" a57="v" a58="v" a59="v" a60="v" a61="v" a62="v" a63="v" a64="v" a65="v" a66="v" a67="v" a68="v" a69="v">
|
||||
4
fuzz/corpus/singlefile/rawtext.html
Normal file
4
fuzz/corpus/singlefile/rawtext.html
Normal file
@@ -0,0 +1,4 @@
|
||||
<script>var s="</scripting>"; if(a</b) x=1;</script>
|
||||
<script src="j.js"></script>
|
||||
<textarea></textareas></textarea>
|
||||
<title></titles></title>
|
||||
2
fuzz/corpus/singlefile/srcset.html
Normal file
2
fuzz/corpus/singlefile/srcset.html
Normal file
@@ -0,0 +1,2 @@
|
||||
<img srcset="a.png 1x, big.png 2x, a.png 100w">
|
||||
<source srcset="a.png,, a.png 2x," src="a.png">
|
||||
2
fuzz/corpus/singlefile/style-attr.html
Normal file
2
fuzz/corpus/singlefile/style-attr.html
Normal file
@@ -0,0 +1,2 @@
|
||||
<div style="background:url(a.png);list-style:url('a.png')"></div>
|
||||
<p style='background:url("a.png")'>x</p>
|
||||
5
fuzz/corpus/singlefile/style-block.html
Normal file
5
fuzz/corpus/singlefile/style-block.html
Normal file
@@ -0,0 +1,5 @@
|
||||
<style>@import "s.css";
|
||||
@import url(sub/b.css);
|
||||
div{background:url(a.png)}
|
||||
/* url(a.png) */ p:after{content:"url(a.png)"}
|
||||
</style>
|
||||
131
fuzz/fuzz-singlefile.c
Normal file
131
fuzz/fuzz-singlefile.c
Normal file
@@ -0,0 +1,131 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the --single-file rewriter (htssinglefile.c): hostile HTML walked
|
||||
through the tag, CSS url()/@import and srcset parsers, then re-serialized.
|
||||
The resolver is aimed at a private temp tree, so the inlining half (MIME
|
||||
guess, base64, nested stylesheet) is reached and nothing else on disk is. */
|
||||
#include "fuzz.h"
|
||||
|
||||
#include "httrack-library.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* Between a.png and big.png, so one input reaches both the inline path and the
|
||||
over-cap fallback. */
|
||||
#define FUZZ_SF_CAP 64
|
||||
|
||||
static char sf_root[512];
|
||||
static char sf_page[600];
|
||||
|
||||
/* The asset tree, in removal order: the subdirectory comes after its file. */
|
||||
static const char *const sf_files[] = {"a.png", "big.png", "j.js", "s.css",
|
||||
"sub/b.css", "sub", NULL};
|
||||
|
||||
static void sf_cleanup(void) {
|
||||
char path[700];
|
||||
int i;
|
||||
|
||||
for (i = 0; sf_files[i] != NULL; i++) {
|
||||
snprintf(path, sizeof(path), "%s/%s", sf_root, sf_files[i]);
|
||||
(void) remove(path);
|
||||
}
|
||||
(void) remove(sf_root);
|
||||
}
|
||||
|
||||
/* A missing asset would silently reduce the target to its parser half. */
|
||||
static void sf_write(const char *name, const char *data, size_t len) {
|
||||
char path[700];
|
||||
FILE *fp;
|
||||
|
||||
snprintf(path, sizeof(path), "%s/%s", sf_root, name);
|
||||
fp = fopen(path, "wb");
|
||||
if (fp == NULL || fwrite(data, 1, len, fp) != len)
|
||||
abort();
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
static void sf_text(const char *name, const char *data) {
|
||||
sf_write(name, data, strlen(data));
|
||||
}
|
||||
|
||||
static void sf_init(void) {
|
||||
static const char png[] = "\x89PNG\r\n\x1a\n";
|
||||
static const char big[4096] = "\x89PNG";
|
||||
const char *tmp = getenv("TMPDIR");
|
||||
char path[700];
|
||||
|
||||
hts_init();
|
||||
snprintf(sf_root, sizeof(sf_root), "%s/httrack-fuzz-sf-XXXXXX",
|
||||
tmp != NULL && tmp[0] != '\0' ? tmp : "/tmp");
|
||||
if (mkdtemp(sf_root) == NULL)
|
||||
abort();
|
||||
atexit(sf_cleanup);
|
||||
snprintf(sf_page, sizeof(sf_page), "%s/page.html", sf_root);
|
||||
snprintf(path, sizeof(path), "%s/sub", sf_root);
|
||||
if (mkdir(path, 0700) != 0)
|
||||
abort();
|
||||
sf_write("a.png", png, sizeof(png) - 1);
|
||||
sf_write("big.png", big, sizeof(big));
|
||||
sf_text("j.js", "var x=1;\n");
|
||||
/* @import plus a url(), so an inlined stylesheet recurses and its own
|
||||
relative reference is rebased. */
|
||||
sf_text("s.css", "@import url(sub/b.css);\ndiv{background:url(a.png)}\n");
|
||||
sf_text("sub/b.css", "p{background:url(../a.png)}\n");
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static int inited = 0;
|
||||
|
||||
String out = STRING_EMPTY;
|
||||
httrackp *opt;
|
||||
/* Exact-length, unterminated: the rewriter is span-based, so ASan bounds a
|
||||
read past html_len instead of it landing on a terminator. */
|
||||
char *html = malloct(size != 0 ? size : 1);
|
||||
|
||||
if (!inited) {
|
||||
sf_init();
|
||||
inited = 1;
|
||||
}
|
||||
memcpy(html, data, size);
|
||||
|
||||
opt = hts_create_opt();
|
||||
opt->log = opt->errlog = NULL;
|
||||
opt->single_file_max_size = FUZZ_SF_CAP;
|
||||
|
||||
StringClear(out);
|
||||
(void) singlefile_rewrite_html(opt, sf_root, sf_page, html, size,
|
||||
SINGLEFILE_MAX_PAGE_SIZE, &out);
|
||||
|
||||
StringFree(out);
|
||||
freet(html);
|
||||
hts_free_opt(opt);
|
||||
return 0;
|
||||
}
|
||||
@@ -300,6 +300,7 @@ ones it kept so the local copy browses offline. These options tune both halves.<
|
||||
<tr><td><tt>--preserve (-%p), --disable-passwords (-%x)</tt></td><td>Leave HTML untouched (no rewriting), and strip passwords out of saved links.</td></tr>
|
||||
<tr><td><tt>--extended-parsing (-%P), --parse-java (-j)</tt></td><td>Aggressive link discovery, and how much script content is parsed for links.</td></tr>
|
||||
<tr><td><tt>--mime-html (-%M)</tt></td><td>Save the whole mirror as a single MIME-encapsulated <tt>.mht</tt> archive (<tt>index.mht</tt>).</td></tr>
|
||||
<tr><td><tt>--single-file (-%Z), --single-file-max-size N</tt></td><td>Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded as <tt>data:</tt> URIs. Assets over the cap (10 MB by default) keep their link, as do audio, video, and the links from one page to another. A sibling of <tt>-%M</tt>, not a replacement: see the recipe below for which to pick.</td></tr>
|
||||
<tr><td><tt>--index (-I), --build-top-index (-%i), --search-index (-%I)</tt></td><td>Build a per-mirror index, a top index across projects, and a searchable keyword index.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -491,6 +492,25 @@ with the same content it served last time lands in <tt>unchanged</tt>. Deletions
|
||||
are reported whether or not <tt>--purge-old</tt> is deleting them. The format is
|
||||
documented in <a href="changes.html">the change report specification</a>.</small></p>
|
||||
|
||||
<h4>Pages you can hand to someone as one file</h4>
|
||||
<p><tt>httrack https://example.com/ --single-file --path mydir</tt><br>
|
||||
<small>Rewrites each saved page after the crawl so its stylesheets, scripts,
|
||||
images and fonts are embedded as <tt>data:</tt> URIs. The mirror stays a normal
|
||||
browsable tree, with links between pages relative and the assets still on disk,
|
||||
but any single <tt>.html</tt> file can now be mailed or archived on its own.
|
||||
Raise or lower the 10 MB per-asset limit with
|
||||
<tt>--single-file-max-size N</tt>; anything over it, plus audio and video, keeps
|
||||
its link. One caveat if you raise it: an inlined stylesheet becomes a
|
||||
<tt>data:</tt> URL, whose path is opaque, so an asset it referenced that stayed
|
||||
over the cap no longer resolves from inside it. Raising the cap past that asset
|
||||
embeds it too and the question goes away.<br>
|
||||
Reach for this when the file has to open for someone you cannot make assumptions
|
||||
about: it is plain HTML and needs no add-on. Reach for <tt>--mime-html</tt>
|
||||
instead when a Chromium-family browser is a given and the mirror is large: MIME
|
||||
carries text parts without the base64 tax, keeps each resource's original URL,
|
||||
and stores a shared asset once rather than re-embedding it in every page that
|
||||
uses it.</small></p>
|
||||
|
||||
<h4>HTTrack as a fetch tool</h4>
|
||||
<p><tt>httrack --get https://host/file.bin --path tmp</tt><br>
|
||||
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
|
||||
|
||||
@@ -90,9 +90,10 @@ offline browser : copy websites to a local directory</p>
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
<b>-%x, --disable-passwords</b> ] [ <b>-%q,
|
||||
] [ <b>-%Z, --single-file</b> ] [ <b>-LN,
|
||||
--long-names[=N]</b> ] [ <b>-KN, --keep-links[=N]</b> ] [
|
||||
<b>-x, --replace-external</b> ] [ <b>-%x,
|
||||
--disable-passwords</b> ] [ <b>-%q,
|
||||
--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
|
||||
<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
|
||||
] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
|
||||
@@ -649,6 +650,24 @@ don’t wait) (--cached-delayed-type-check)</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%Z</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>after the mirror, rewrite each saved page with its
|
||||
stylesheets, scripts, images and fonts inlined as data:
|
||||
URIs, so any page opens by double-click anywhere (links
|
||||
between pages stay relative; audio and video stay links);
|
||||
--single-file-max-size N caps each asset (default 10485760
|
||||
bytes). %M is the better container where a Chromium-family
|
||||
browser is a given: one archive, no base64 tax on text, a
|
||||
shared asset stored once (--single-file)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%t</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
@@ -104,6 +104,7 @@ ${do:end-if}
|
||||
<input type="hidden" name="hidepwd" value="">
|
||||
<input type="hidden" name="hidequery" value="">
|
||||
<input type="hidden" name="nopurge" value="">
|
||||
<input type="hidden" name="singlefile" value="">
|
||||
|
||||
${LANG_I33}
|
||||
<br>
|
||||
@@ -143,6 +144,13 @@ ${listid:build:LISTDEF_3}
|
||||
<tr><td><input type="checkbox" name="nopurge" ${checked:nopurge}
|
||||
title='${html:LANG_I1a}' onMouseOver="info('${html:LANG_I1a}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I57}</td></tr>
|
||||
<tr><td><input type="checkbox" name="singlefile" ${checked:singlefile}
|
||||
title='${html:LANG_SINGLEFILETIP}' onMouseOver="info('${html:LANG_SINGLEFILETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_SINGLEFILE}</td></tr>
|
||||
<tr><td>${LANG_SINGLEFILEMAX}
|
||||
<input name="singlefilemax" value="${singlefilemax}" size="12"
|
||||
title='${html:LANG_SINGLEFILEMAXTIP}' onMouseOver="info('${html:LANG_SINGLEFILEMAXTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
></td></tr>
|
||||
</table>
|
||||
|
||||
<tr><td>
|
||||
|
||||
@@ -144,6 +144,8 @@ ${do:copy:StoreAllInCache:cache2}
|
||||
${do:copy:Warc:warc}
|
||||
${do:copy:WarcFile:warcfile}
|
||||
${do:copy:Changes:changes}
|
||||
${do:copy:SingleFile:singlefile}
|
||||
${do:copy:SingleFileMaxSize:singlefilemax}
|
||||
${do:copy:LogType:logtype}
|
||||
${do:copy:UseHTTPProxyForFTP:ftpprox}
|
||||
${do:copy:ProxyType:proxytype}
|
||||
|
||||
@@ -191,6 +191,8 @@ ${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one *
|
||||
${test:warc:--warc}
|
||||
${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
|
||||
${test:changes:--changes}
|
||||
${test:singlefile:--single-file}
|
||||
${test:singlefilemax:--single-file-max-size=}${unquoted:singlefilemax}
|
||||
${test:norecatch:--do-not-recatch}
|
||||
${test:logf:--single-log}
|
||||
${test:logtype:::--extra-log:--debug-log}
|
||||
@@ -244,6 +246,8 @@ StoreAllInCache=${ztest:cache2:0:1}
|
||||
Warc=${ztest:warc:0:1}
|
||||
WarcFile=${warcfile}
|
||||
Changes=${ztest:changes:0:1}
|
||||
SingleFile=${ztest:singlefile:0:1}
|
||||
SingleFileMaxSize=${singlefilemax}
|
||||
LogType=${logtype}
|
||||
UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
|
||||
ProxyType=${proxytype}
|
||||
|
||||
8
lang.def
8
lang.def
@@ -1046,3 +1046,11 @@ LANG_CHANGES
|
||||
Report what changed since the previous mirror
|
||||
LANG_CHANGESTIP
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
LANG_SINGLEFILE
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
LANG_SINGLEFILETIP
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
LANG_SINGLEFILEMAX
|
||||
Largest inlined asset (bytes):
|
||||
LANG_SINGLEFILEMAXTIP
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Îò÷åò çà ïðîìåíèòå ñïðÿìî ïðåäèøíîòî îãëåäàëî
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Çàïèñâàíå è íà hts-changes.json ñúñ ñïèñúê íà íîâèòå, ïðîìåíåíèòå, íåïðîìåíåíèòå è èç÷åçíàëèòå ôàéëîâå ñïðÿìî ïðåäèøíîòî îãëåäàëî.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Âãðàæäàíå íà ðåñóðñèòå êàòî data: URI (ñàìîñòîÿòåëíè ñòðàíèöè)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ñëåä çàâúðøâàíå íà îãëåäàëîòî âñÿêà çàïàçåíà ñòðàíèöà ñå ïðåçàïèñâà ñ âãðàäåíè ñòèëîâå, ñêðèïòîâå, èçîáðàæåíèÿ è øðèôòîâå, òàêà ÷å ñòðàíèöàòà äà ìîæå äà ñå îòâîðè è ñàìîñòîÿòåëíî.
|
||||
Largest inlined asset (bytes):
|
||||
Íàé-ãîëÿì âãðàäåí ðåñóðñ (áàéòîâå):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Ðåñóðñ íàä òîçè ðàçìåð çàïàçâà îáèêíîâåíà âðúçêà; îñòàâåòå ïðàçíî çà ñòîéíîñòòà ïî ïîäðàçáèðàíå îò 10485760 áàéòà.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Informar de los cambios desde la copia anterior
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Escribir también hts-changes.json con la lista de lo que esta captura deja como nuevo, modificado, sin cambios o desaparecido respecto a la copia anterior.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incrustar los recursos como URI data: (páginas autónomas)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Una vez terminada la copia, reescribir cada página guardada con sus hojas de estilo, scripts, imágenes y fuentes incrustadas, de modo que una página también pueda abrirse por sí sola.
|
||||
Largest inlined asset (bytes):
|
||||
Tamaño máximo del recurso incrustado (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Un recurso mayor que este tamaño conserva un enlace normal; déjelo vacío para el valor predeterminado de 10485760 bytes.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Hlásit, co se od pøedchozího zrcadlení zmìnilo
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Zapsat také hts-changes.json se seznamem toho, co je oproti pøedchozímu zrcadlení nové, zmìnìné, nezmìnìné nebo chybìjící.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Vložit zdroje jako URI data: (samostatné stránky)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Po dokonèení zrcadlení pøepsat každou uloženou stránku s vloženými styly, skripty, obrázky a písmy, aby ji bylo možné otevøít i samostatnì.
|
||||
Largest inlined asset (bytes):
|
||||
Nejvìtší vložený zdroj (bajty):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Zdroj vìtší než tato velikost si ponechá bìžný odkaz; ponechte prázdné pro výchozí hodnotu 10485760 bajtù.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
回報自上次鏡射以來的變更
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
同時寫入 hts-changes.json,列出這次擷取相對於上次鏡射的新增、變更、未變更與消失的項目。
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
將資源內嵌為 data: URI(獨立網頁)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
鏡射完成後,重寫每個已儲存的網頁,將樣式表、指令碼、圖片與字型內嵌其中,讓網頁也能單獨開啟。
|
||||
Largest inlined asset (bytes):
|
||||
內嵌資源大小上限(位元組):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
超過此大小的資源會保留一般連結;留空則使用預設的 10485760 位元組。
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
报告自上次镜像以来的变更
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
同时写入 hts-changes.json,列出本次抓取相对于上次镜像的新增、更改、未更改和消失的项目。
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
将资源内嵌为 data: URI(独立网页)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
镜像完成后,重写每个已保存的网页,将样式表、脚本、图片和字体内嵌其中,使网页也能单独打开。
|
||||
Largest inlined asset (bytes):
|
||||
内嵌资源大小上限(字节):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
超过此大小的资源会保留普通链接;留空则使用默认的 10485760 字节。
|
||||
|
||||
@@ -970,3 +970,11 @@ Report what changed since the previous mirror
|
||||
Prijavi ¹to se promijenilo od prethodnog zrcaljenja
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Zapi¹i i hts-changes.json s popisom onoga ¹to je u odnosu na prethodno zrcaljenje novo, promijenjeno, nepromijenjeno ili nestalo.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Ugradi resurse kao data: URI-je (samostalne stranice)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Nakon dovr¹etka zrcaljenja ponovno zapi¹i svaku spremljenu stranicu s ugraðenim stilskim listovima, skriptama, slikama i fontovima, tako da se stranica mo¾e otvoriti i zasebno.
|
||||
Largest inlined asset (bytes):
|
||||
Najveæi ugraðeni resurs (bajtovi):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Resurs veæi od ove velièine zadr¾ava obiènu poveznicu; ostavite prazno za zadanih 10485760 bajtova.
|
||||
|
||||
@@ -1016,3 +1016,11 @@ Report what changed since the previous mirror
|
||||
Rapportér hvad der er ændret siden den forrige spejling
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Skriv også hts-changes.json med en liste over, hvad denne gennemgang efterlader som nyt, ændret, uændret eller forsvundet i forhold til den forrige spejling.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Indlejr ressourcer som data:-URI'er (selvstændige sider)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Når spejlingen er færdig, omskrives hver gemt side med dens typografiark, scripts, billeder og skrifttyper indlejret, så en side også kan åbnes alene.
|
||||
Largest inlined asset (bytes):
|
||||
Største indlejrede ressource (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
En ressource over denne størrelse beholder et almindeligt link; lad feltet stå tomt for standardværdien på 10485760 byte.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Melden, was sich seit der vorherigen Spiegelung geändert hat
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Zusätzlich hts-changes.json schreiben, das auflistet, was dieser Durchlauf gegenüber der vorherigen Spiegelung als neu, geändert, unverändert oder verschwunden hinterlässt.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Ressourcen als data:-URIs einbetten (eigenständige Seiten)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Nach Abschluss der Spiegelung jede gespeicherte Seite mit eingebetteten Stylesheets, Skripten, Bildern und Schriftarten neu schreiben, sodass eine Seite auch für sich allein geöffnet werden kann.
|
||||
Largest inlined asset (bytes):
|
||||
Größte eingebettete Ressource (Bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Eine Ressource über dieser Größe behält einen gewöhnlichen Link; leer lassen für den Standardwert von 10485760 Bytes.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Teata, mis on eelmisest peegeldusest saadik muutunud
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Kirjuta ka hts-changes.json, mis loetleb, mis on võrreldes eelmise peegeldusega uus, muutunud, muutumatu või kadunud.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Manusta ressursid data: URI-dena (iseseisvad lehed)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Pärast peegelduse lõppu kirjuta iga salvestatud leht ümber, manustades selle laaditabelid, skriptid, pildid ja fondid, nii et lehte saab avada ka eraldi.
|
||||
Largest inlined asset (bytes):
|
||||
Suurim manustatud ressurss (baiti):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Sellest suurem ressurss säilitab tavalise lingi; jäta tühjaks vaikeväärtuse 10485760 baiti jaoks.
|
||||
|
||||
@@ -1016,3 +1016,11 @@ Report what changed since the previous mirror
|
||||
Report what changed since the previous mirror
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Largest inlined asset (bytes):
|
||||
Largest inlined asset (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -970,3 +970,11 @@ Report what changed since the previous mirror
|
||||
Raportoi, mikä on muuttunut edellisen peilauksen jälkeen
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Kirjoita myös hts-changes.json, joka luettelee, mikä on edelliseen peilaukseen verrattuna uutta, muuttunutta, muuttumatonta tai kadonnutta.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Upota resurssit data:-URI-osoitteina (itsenäiset sivut)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Kun peilaus on valmis, kirjoita jokainen tallennettu sivu uudelleen niin, että sen tyylitiedostot, komentosarjat, kuvat ja fontit upotetaan, jolloin sivun voi avata myös yksinään.
|
||||
Largest inlined asset (bytes):
|
||||
Suurin upotettu resurssi (tavua):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Tätä suurempi resurssi säilyttää tavallisen linkin; jätä tyhjäksi, jolloin käytetään oletusarvoa 10485760 tavua.
|
||||
|
||||
@@ -1016,3 +1016,11 @@ Report what changed since the previous mirror
|
||||
Signaler ce qui a changé depuis le miroir précédent
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Écrire aussi hts-changes.json, qui liste ce que ce crawl laisse nouveau, modifié, inchangé ou disparu par rapport au miroir précédent.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Intégrer les ressources en URI data: (pages autonomes)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Une fois le miroir terminé, réécrire chaque page enregistrée avec ses feuilles de style, scripts, images et polices intégrés, afin qu'une page puisse aussi être ouverte seule.
|
||||
Largest inlined asset (bytes):
|
||||
Taille maximale d'une ressource intégrée (octets) :
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Au-delà de cette taille, la ressource reste un lien ordinaire ; laissez vide pour la valeur par défaut de 10485760 octets.
|
||||
|
||||
@@ -970,3 +970,11 @@ Report what changed since the previous mirror
|
||||
Αναφορά των αλλαγών από το προηγούμενο αντίγραφο
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Εγγραφή και του hts-changes.json, που παραθέτει τι είναι νέο, τι άλλαξε, τι έμεινε ίδιο και τι χάθηκε σε σχέση με το προηγούμενο αντίγραφο.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Ενσωμάτωση των πόρων ως URI data: (αυτόνομες σελίδες)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Μόλις ολοκληρωθεί το αντίγραφο, κάθε αποθηκευμένη σελίδα ξαναγράφεται με ενσωματωμένα τα φύλλα στυλ, τα σενάρια, τις εικόνες και τις γραμματοσειρές της, ώστε η σελίδα να μπορεί να ανοίξει και μόνη της.
|
||||
Largest inlined asset (bytes):
|
||||
Μέγιστος ενσωματωμένος πόρος (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Πόρος μεγαλύτερος από αυτό το μέγεθος διατηρεί κανονικό σύνδεσμο. Αφήστε το κενό για την προεπιλογή των 10485760 byte.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Segnala che cosa è cambiato dalla copia precedente
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Scrive anche hts-changes.json, che elenca ciò che questa scansione lascia come nuovo, modificato, invariato o scomparso rispetto alla copia precedente.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incorpora le risorse come URI data: (pagine autonome)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Al termine della copia, riscrive ogni pagina salvata incorporando fogli di stile, script, immagini e caratteri, in modo che una pagina possa essere aperta anche da sola.
|
||||
Largest inlined asset (bytes):
|
||||
Dimensione massima della risorsa incorporata (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Una risorsa oltre questa dimensione mantiene un collegamento normale; lasciare vuoto per il valore predefinito di 10485760 byte.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
前回のミラーからの変更点を報告する
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
hts-changes.json も書き出し、前回のミラーと比べて新規、変更、変更なし、消滅となったものを一覧にします。
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
リソースを data: URI として埋め込む (単独で開けるページ)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
ミラーの完了後、保存された各ページを、スタイルシート、スクリプト、画像、フォントを埋め込んだ形で書き直します。ページ単体でも開けるようになります。
|
||||
Largest inlined asset (bytes):
|
||||
埋め込む最大サイズ (バイト):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
このサイズを超えるリソースは通常のリンクのままになります。空欄にすると既定値の 10485760 バイトになります。
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
¸×ÒÕáâØ èâÞ Õ ßàÞÜÕÝÕâÞ ÞÔ ßàÕâåÞÔÝÞâÞ ÞÓÛÕÔÐÛÞ
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
·ÐßØèØ Ø hts-changes.json áÞ áߨáÞÚ ÝÐ âÞÐ èâÞ Õ ÝÞÒÞ, ßàÞÜÕÝÕâÞ, ÝÕßàÞÜÕÝÕâÞ ØÛØ ØáçÕ×ÝÐâÞ ÒÞ ÞÔÝÞá ÝÐ ßàÕâåÞÔÝÞâÞ ÞÓÛÕÔÐÛÞ.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
²ÓàÐÔØ ÓØ àÕáãàáØâÕ ÚÐÚÞ data: URI (áÐÜÞáâÞøÝØ áâàÐÝØæØ)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
¿Þ ×ÐÒàèãÒÐúÕ ÝÐ ÞÓÛÕÔÐÛÞâÞ, áÕÚÞøÐ ×ÐçãÒÐÝÐ áâàÐÝØæÐ áÕ ßàÕߨèãÒÐ áÞ ÒÓàÐÔÕÝØ áâØÛáÚØ ÛØáâÞÒØ, áÚàØßâØ, áÛØÚØ Ø äÞÝâÞÒØ, ×Ð ÔÐ ÜÞÖÕ áâàÐÝØæÐâÐ ÔÐ áÕ ÞâÒÞàØ Ø áÐÜÞáâÞøÝÞ.
|
||||
Largest inlined asset (bytes):
|
||||
½ÐøÓÞÛÕÜ ÒÓàÐÔÕÝ àÕáãàá (ÑÐøâØ):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
ÀÕáãàá ßÞÓÞÛÕÜ ÞÔ ÞÒÐÐ ÓÞÛÕÜØÝÐ ×ÐÔàÖãÒÐ ÞÑØçÝÐ ÒàáÚÐ; ÞáâÐÒÕâÕ ßàÐ×ÝÞ ×Ð áâÐÝÔÐàÔÝØâÕ 10485760 ÑÐøâØ.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Jelentés arról, mi változott az elõzõ tükrözés óta
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
A hts-changes.json fájl írása is, amely felsorolja, mi új, mi változott, mi maradt változatlan és mi tûnt el az elõzõ tükrözéshez képest.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Erõforrások beágyazása data: URI-ként (önálló oldalak)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
A tükrözés befejezése után minden mentett oldal újraírása a stíluslapok, parancsfájlok, képek és betûtípusok beágyazásával, így az oldal önmagában is megnyitható.
|
||||
Largest inlined asset (bytes):
|
||||
Legnagyobb beágyazott erõforrás (bájt):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Az ennél nagyobb erõforrás közönséges hivatkozás marad; hagyja üresen a 10485760 bájtos alapértelmezéshez.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Rapporteren wat er sinds de vorige spiegeling is gewijzigd
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Ook hts-changes.json schrijven met een overzicht van wat deze doorloop nieuw, gewijzigd, ongewijzigd of verdwenen laat ten opzichte van de vorige spiegeling.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Bronnen insluiten als data:-URI's (op zichzelf staande pagina's)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Zodra de spiegeling klaar is, elke opgeslagen pagina herschrijven met ingesloten stijlbladen, scripts, afbeeldingen en lettertypen, zodat een pagina ook los geopend kan worden.
|
||||
Largest inlined asset (bytes):
|
||||
Grootste ingesloten bron (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Een bron boven deze grootte houdt een gewone koppeling; laat leeg voor de standaardwaarde van 10485760 bytes.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Rapporter hva som er endret siden forrige speiling
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Skriv også hts-changes.json som lister opp hva denne gjennomgangen etterlater som nytt, endret, uendret eller forsvunnet sammenlignet med forrige speiling.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Bygg inn ressurser som data:-URI-er (selvstendige sider)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Når speilingen er ferdig, skrives hver lagrede side om med stilark, skript, bilder og skrifter innebygd, slik at en side også kan åpnes alene.
|
||||
Largest inlined asset (bytes):
|
||||
Største innebygde ressurs (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
En ressurs over denne størrelsen beholder en vanlig lenke; la feltet stå tomt for standardverdien på 10485760 byte.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Zg³o¶, co zmieni³o siê od poprzedniej kopii
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Zapisz równie¿ plik hts-changes.json z list± tego, co w porównaniu z poprzedni± kopi± jest nowe, zmienione, niezmienione lub usuniête.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Osad¼ zasoby jako identyfikatory data: URI (samodzielne strony)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Po zakoñczeniu tworzenia kopii przepisz ka¿d± zapisan± stronê z osadzonymi arkuszami stylów, skryptami, obrazami i czcionkami, aby stronê mo¿na by³o otworzyæ równie¿ samodzielnie.
|
||||
Largest inlined asset (bytes):
|
||||
Najwiêkszy osadzony zasób (bajty):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Zasób wiêkszy ni¿ ten rozmiar zachowuje zwyk³y odno¶nik; pozostaw puste, aby u¿yæ domy¶lnych 10485760 bajtów.
|
||||
|
||||
@@ -1016,3 +1016,11 @@ Report what changed since the previous mirror
|
||||
Relatar o que mudou desde o espelhamento anterior
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Gravar também hts-changes.json listando o que esta captura deixa como novo, alterado, inalterado ou removido em relação ao espelhamento anterior.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incorporar os recursos como URIs data: (páginas autônomas)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ao terminar o espelhamento, reescrever cada página salva com suas folhas de estilo, scripts, imagens e fontes incorporadas, para que a página também possa ser aberta sozinha.
|
||||
Largest inlined asset (bytes):
|
||||
Maior recurso incorporado (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Um recurso acima desse tamanho mantém um link comum; deixe em branco para o padrão de 10485760 bytes.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Comunicar o que mudou desde o espelho anterior
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Escrever também hts-changes.json, que lista o que esta recolha deixa como novo, alterado, inalterado ou desaparecido em relação ao espelho anterior.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incorporar os recursos como URI data: (páginas autónomas)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Depois de terminado o espelho, reescrever cada página guardada com as suas folhas de estilo, scripts, imagens e tipos de letra incorporados, para que a página também possa ser aberta sozinha.
|
||||
Largest inlined asset (bytes):
|
||||
Maior recurso incorporado (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Um recurso acima deste tamanho mantém uma ligação normal; deixe em branco para o valor predefinido de 10485760 bytes.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Raporteaza ce s-a schimbat fata de copia anterioara
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Scrie si hts-changes.json, care listeaza ce este nou, modificat, nemodificat sau disparut fata de copia anterioara.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Încorporeaza resursele ca URI data: (pagini de sine statatoare)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Dupa terminarea copiei, fiecare pagina salvata este rescrisa cu foile de stil, scripturile, imaginile si fonturile încorporate, astfel încât pagina sa poata fi deschisa si singura.
|
||||
Largest inlined asset (bytes):
|
||||
Cea mai mare resursa încorporata (octeti):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
O resursa mai mare decât aceasta dimensiune pastreaza o legatura obisnuita; lasati gol pentru valoarea implicita de 10485760 de octeti.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Ñîîáùàòü, ÷òî èçìåíèëîñü ñ ïðåäûäóùåãî çåðêàëà
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Òàêæå çàïèñûâàòü hts-changes.json ñî ñïèñêîì òîãî, ÷òî ïî ñðàâíåíèþ ñ ïðåäûäóùèì çåðêàëîì ñòàëî íîâûì, èçìåí¸ííûì, íåèçìåí¸ííûì èëè èñ÷åçëî.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Âñòðàèâàòü ðåñóðñû êàê data: URI (ñàìîñòîÿòåëüíûå ñòðàíèöû)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ïîñëå çàâåðøåíèÿ çåðêàëèðîâàíèÿ êàæäàÿ ñîõðàí¸ííàÿ ñòðàíèöà ïåðåçàïèñûâàåòñÿ ñî âñòðîåííûìè òàáëèöàìè ñòèëåé, ñöåíàðèÿìè, èçîáðàæåíèÿìè è øðèôòàìè, ÷òîáû ñòðàíèöó ìîæíî áûëî îòêðûòü îòäåëüíî.
|
||||
Largest inlined asset (bytes):
|
||||
Íàèáîëüøèé âñòðàèâàåìûé ðåñóðñ (áàéòû):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Ðåñóðñ áîëüøå ýòîãî ðàçìåðà ñîõðàíÿåò îáû÷íóþ ññûëêó; îñòàâüòå ïóñòûì äëÿ çíà÷åíèÿ ïî óìîë÷àíèþ 10485760 áàéò.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Oznámi», èo sa od predchádzajúceho zrkadlenia zmenilo
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Zapísa» aj hts-changes.json so zoznamom toho, èo je oproti predchádzajúcemu zrkadleniu nové, zmenené, nezmenené alebo chýbajúce.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Vlo¾i» zdroje ako URI data: (samostatné stránky)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Po dokonèení zrkadlenia prepísa» ka¾dú ulo¾enú stránku s vlo¾enými ¹týlmi, skriptmi, obrázkami a písmami, aby sa stránka dala otvori» aj samostatne.
|
||||
Largest inlined asset (bytes):
|
||||
Najväè¹í vlo¾ený zdroj (bajty):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Zdroj väè¹í ne¾ táto veµkos» si ponechá be¾ný odkaz; ponechajte prázdne pre predvolených 10485760 bajtov.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Porocaj, kaj se je spremenilo od prejsnjega zrcaljenja
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Zapisi tudi hts-changes.json s seznamom tega, kar je v primerjavi s prejsnjim zrcaljenjem novo, spremenjeno, nespremenjeno ali izginilo.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Vgradi vire kot data: URI (samostojne strani)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ko je zrcaljenje koncano, se vsaka shranjena stran prepise z vgrajenimi slogovnimi listi, skripti, slikami in pisavami, tako da jo je mogoce odpreti tudi samostojno.
|
||||
Largest inlined asset (bytes):
|
||||
Najvecji vgrajeni vir (bajti):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Vir, vecji od te velikosti, ohrani obicajno povezavo; pustite prazno za privzetih 10485760 bajtov.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Rapportera vad som ändrats sedan föregående spegling
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Skriv även hts-changes.json som listar vad den här genomgången lämnar som nytt, ändrat, oförändrat eller försvunnet jämfört med föregående spegling.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Bädda in resurser som data:-URI:er (fristående sidor)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
När speglingen är klar skrivs varje sparad sida om med sina formatmallar, skript, bilder och teckensnitt inbäddade, så att en sida också kan öppnas för sig.
|
||||
Largest inlined asset (bytes):
|
||||
Största inbäddade resurs (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
En resurs över den här storleken behåller en vanlig länk; lämna tomt för standardvärdet 10485760 byte.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Önceki yansýmadan bu yana deðiþenleri bildir
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Ayrýca hts-changes.json yazarak bu taramanýn önceki yansýmaya göre neyi yeni, deðiþmiþ, deðiþmemiþ veya kaybolmuþ býraktýðýný listele.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Kaynaklarý data: URI olarak göm (kendi kendine yeten sayfalar)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Yansýlama bittiðinde, kaydedilen her sayfa stil sayfalarý, betikleri, görselleri ve yazý tipleri gömülü olarak yeniden yazýlýr; böylece sayfa tek baþýna da açýlabilir.
|
||||
Largest inlined asset (bytes):
|
||||
En büyük gömülü kaynak (bayt):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Bu boyutun üzerindeki bir kaynak sýradan baðlantýsýný korur; 10485760 baytlýk varsayýlan için boþ býrakýn.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Ïîâ³äîìëÿòè, ùî çì³íèëîñÿ ç ïîïåðåäíüîãî äçåðêàëà
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
Òàêîæ çàïèñóâàòè hts-changes.json ç³ ñïèñêîì òîãî, ùî ïîð³âíÿíî ç ïîïåðåäí³ì äçåðêàëîì º íîâèì, çì³íåíèì, íåçì³íåíèì àáî çíèêëèì.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Âáóäîâóâàòè ðåñóðñè ÿê data: URI (ñàìîñò³éí³ ñòîð³íêè)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
ϳñëÿ çàâåðøåííÿ äçåðêàëþâàííÿ êîæíà çáåðåæåíà ñòîð³íêà ïåðåçàïèñóºòüñÿ ç âáóäîâàíèìè òàáëèöÿìè ñòèë³â, ñöåíàð³ÿìè, çîáðàæåííÿìè òà øðèôòàìè, ùîá ñòîð³íêó ìîæíà áóëî â³äêðèòè îêðåìî.
|
||||
Largest inlined asset (bytes):
|
||||
Íàéá³ëüøèé âáóäîâàíèé ðåñóðñ (áàéòè):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Ðåñóðñ, á³ëüøèé çà öåé ðîçì³ð, çáåð³ãຠçâè÷àéíå ïîñèëàííÿ; çàëèøòå ïîðîæí³ì äëÿ òèïîâîãî çíà÷åííÿ 10485760 áàéò³â.
|
||||
|
||||
@@ -968,3 +968,11 @@ Report what changed since the previous mirror
|
||||
Oldingi nusxadan beri nima o’zgarganini xabar qilish
|
||||
Also write hts-changes.json listing what this crawl left new, changed, unchanged and gone compared to the previous mirror.
|
||||
hts-changes.json ham yoziladi: unda ushbu yig’ish oldingi nusxaga nisbatan nimani yangi, o’zgargan, o’zgarmagan yoki yo’qolgan holda qoldirgani ro’yxati bo’ladi.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Resurslarni data: URI sifatida joylash (mustaqil sahifalar)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Nusxalash tugagach, har bir saqlangan sahifa uslublar jadvallari, skriptlar, rasmlar va shriftlar joylangan holda qayta yoziladi, shunda sahifani alohida ham ochish mumkin.
|
||||
Largest inlined asset (bytes):
|
||||
Eng katta joylangan resurs (bayt):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Bu o’lchamdan katta resurs oddiy havolani saqlab qoladi; standart 10485760 bayt uchun bo’sh qoldiring.
|
||||
|
||||
@@ -40,6 +40,7 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-%Z, \-\-single\-file\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
[ \fB\-KN, \-\-keep\-links[=N]\fR ]
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
@@ -199,6 +200,8 @@ delayed type check, don't make any link test but wait for files download to star
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%Z
|
||||
after the mirror, rewrite each saved page with its stylesheets, scripts, images and fonts inlined as data: URIs, so any page opens by double\-click anywhere (links between pages stay relative; audio and video stay links); \-\-single\-file\-max\-size N caps each asset (default 10485760 bytes). %M is the better container where a Chromium\-family browser is a given: one archive, no base64 tax on text, a shared asset stored once (\-\-single\-file)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
|
||||
@@ -66,7 +66,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htswarc.c htschanges.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmd5.c htscodec.c htswarc.c htschanges.c htssinglefile.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
@@ -77,7 +77,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htschanges.h htsproxy.h htszlib.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htschanges.h htssinglefile.h htsproxy.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
htscharset.h punycode.h htsencoding.h \
|
||||
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
|
||||
|
||||
@@ -125,6 +125,10 @@ const char *hts_optalias[][4] = {
|
||||
{"warc-cdxj", "-%rc", "single", ""},
|
||||
{"wacz", "-%rz", "single",
|
||||
"package the WARC archive, CDXJ index and pages as a WACZ file"},
|
||||
{"single-file", "-%Z", "single",
|
||||
"after the mirror, inline each page's assets as data: URIs"},
|
||||
{"single-file-max-size", "-%Zs", "param1",
|
||||
"per-asset cap for --single-file, in bytes (implies it; default 10485760)"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
|
||||
@@ -1297,12 +1297,17 @@ char *readfile2(const char *fil, LLint * size) {
|
||||
}
|
||||
|
||||
/* Note: utf-8 */
|
||||
char *readfile_utf8(const char *fil) {
|
||||
char *readfile_utf8(const char *fil) { return readfile2_utf8(fil, NULL); }
|
||||
|
||||
/* Note: utf-8 */
|
||||
char *readfile2_utf8(const char *fil, LLint *size) {
|
||||
char *adr = NULL;
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const LLint len = fsize_utf8(fil);
|
||||
const size_t buflen = len >= 0 ? llint_to_size_t(len) : (size_t) -1;
|
||||
|
||||
if (size != NULL)
|
||||
*size = len;
|
||||
if (buflen != (size_t) -1) { // exists, and is addressable (see readfile2)
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), fil), "rb");
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htschanges.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -2206,6 +2207,10 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
// fin purge!
|
||||
|
||||
/* --single-file: inline each page's assets now the tree is final, after the
|
||||
purge has deleted whatever this run dropped. */
|
||||
singlefile_process_mirror(opt);
|
||||
|
||||
// Indexation
|
||||
if (opt->kindex)
|
||||
index_finish(StringBuff(opt->path_html), opt->kindex);
|
||||
@@ -3676,6 +3681,10 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
to->warc_wacz = from->warc_wacz;
|
||||
to->changes = from->changes;
|
||||
|
||||
to->single_file = from->single_file;
|
||||
if (from->single_file_max_size > 0)
|
||||
to->single_file_max_size = from->single_file_max_size;
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
to->pause_max_ms = from->pause_max_ms;
|
||||
|
||||
@@ -415,6 +415,8 @@ char *readfile2(const char *fil, LLint * size);
|
||||
|
||||
char *readfile_utf8(const char *fil);
|
||||
|
||||
char *readfile2_utf8(const char *fil, LLint *size);
|
||||
|
||||
char *readfile_or(const char *fil, const char *defaultdata);
|
||||
|
||||
/* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
|
||||
|
||||
@@ -1803,6 +1803,36 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
break;
|
||||
case 'Z': // single-file: inline each page's assets as data: URIs
|
||||
if (*(com + 1) == 's') { // --single-file-max-size N
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option single-file-max-size needs a blank "
|
||||
"space and a size");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
{ // reject non-numeric/negative/overflow; keep the default
|
||||
char *end;
|
||||
LLint v;
|
||||
|
||||
errno = 0;
|
||||
v = strtoll(argv[na], &end, 10);
|
||||
if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
|
||||
errno != ERANGE && v > 0)
|
||||
opt->single_file_max_size = v;
|
||||
}
|
||||
opt->single_file = HTS_TRUE;
|
||||
} else {
|
||||
opt->single_file = HTS_TRUE;
|
||||
if (*(com + 1) == '0') {
|
||||
opt->single_file = HTS_FALSE;
|
||||
com++;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
|
||||
@@ -535,6 +535,13 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %Z after the mirror, rewrite each saved page with its "
|
||||
"stylesheets, scripts, images and fonts inlined as data: URIs, so "
|
||||
"any page opens by double-click anywhere (links between pages stay "
|
||||
"relative; audio and video stay links); --single-file-max-size N "
|
||||
"caps each asset (default 10485760 bytes). %M is the better "
|
||||
"container where a Chromium-family browser is a given: one archive, "
|
||||
"no base64 tax on text, a shared asset stored once");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
|
||||
@@ -38,6 +38,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htschanges.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -6032,6 +6033,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
opt->changes = HTS_FALSE;
|
||||
opt->changes_state = NULL;
|
||||
opt->single_file = HTS_FALSE;
|
||||
opt->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
|
||||
@@ -551,6 +551,12 @@ struct httrackp {
|
||||
the previous mirror. Tail: ABI */
|
||||
void *changes_state; /**< live change-report accumulator (hts_changes*),
|
||||
engine-owned. Tail: ABI */
|
||||
hts_boolean single_file; /**< --single-file: once the mirror is done, rewrite
|
||||
each saved page with its assets inlined as
|
||||
data: URIs. Tail: ABI */
|
||||
LLint single_file_max_size; /**< --single-file-max-size: per-asset cap in
|
||||
bytes; a bigger asset stays a link.
|
||||
Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
|
||||
@@ -61,6 +61,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsproxy.h"
|
||||
#include "htswarc.h"
|
||||
#include "htschanges.h"
|
||||
#include "htssinglefile.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
@@ -1747,6 +1748,20 @@ static int st_copyopt(httrackp *opt, int argc, char **argv) {
|
||||
if (to->changes != HTS_TRUE)
|
||||
err = 1;
|
||||
|
||||
/* single_file pair: the cap is guarded by >0, so an unset source must not
|
||||
overwrite the default the target already carries */
|
||||
from->single_file = HTS_TRUE;
|
||||
from->single_file_max_size = 4096;
|
||||
to->single_file = HTS_FALSE;
|
||||
to->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
|
||||
copy_htsopt(from, to);
|
||||
if (!to->single_file || to->single_file_max_size != 4096)
|
||||
err = 1;
|
||||
from->single_file_max_size = 0;
|
||||
copy_htsopt(from, to);
|
||||
if (to->single_file_max_size != 4096)
|
||||
err = 1;
|
||||
|
||||
/* #185 pause pair: copied when enabled (max>0), the 0 sentinel skips */
|
||||
from->pause_min_ms = 5000;
|
||||
from->pause_max_ms = 10000;
|
||||
@@ -4882,6 +4897,520 @@ static int st_warc_wacz(httrackp *opt, int argc, char **argv) {
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* --single-file */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
static int sf_err = 0;
|
||||
|
||||
/* Set when the filesystem accepted a ':' in a name; Windows never does. */
|
||||
static hts_boolean sf_colon_ok = HTS_FALSE;
|
||||
|
||||
static void sf_check(int ok, const char *what) {
|
||||
if (!ok) {
|
||||
fprintf(stderr, "singlefile: %s\n", what);
|
||||
sf_err++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Write rel (a '/'-separated path under dir), creating the directories.
|
||||
Returns HTS_FALSE if the name is one the filesystem will not take. */
|
||||
static hts_boolean sf_try_put(const char *dir, const char *rel,
|
||||
const void *data, size_t len) {
|
||||
char BIGSTK path[HTS_URLMAXSIZE * 2];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *fp;
|
||||
|
||||
fconcat(path, sizeof(path), dir, rel);
|
||||
structcheck_utf8(path);
|
||||
fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
|
||||
if (fp == NULL)
|
||||
return HTS_FALSE;
|
||||
assertf(len == 0 || fwrite(data, 1, len, fp) == len);
|
||||
fclose(fp);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
static void sf_put(const char *dir, const char *rel, const void *data,
|
||||
size_t len) {
|
||||
assertf(sf_try_put(dir, rel, data, len));
|
||||
}
|
||||
|
||||
/* Number of times needle occurs in hay. */
|
||||
static int sf_count(const char *hay, const char *needle) {
|
||||
const size_t l = strlen(needle);
|
||||
int n = 0;
|
||||
const char *p = hay;
|
||||
|
||||
while ((p = strstr(p, needle)) != NULL) {
|
||||
n++;
|
||||
p += l;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* The base64 payload following the first occurrence of prefix, up to the first
|
||||
byte outside the base64 alphabet. NULL if prefix is absent. */
|
||||
static const char *sf_payload(const char *hay, const char *prefix,
|
||||
size_t *len) {
|
||||
const char *p = strstr(hay, prefix);
|
||||
size_t n = 0;
|
||||
|
||||
if (p == NULL)
|
||||
return NULL;
|
||||
p += strlen(prefix);
|
||||
while (p[n] != '\0' && (isalnum((unsigned char) p[n]) || p[n] == '+' ||
|
||||
p[n] == '/' || p[n] == '='))
|
||||
n++;
|
||||
*len = n;
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Independent base64 decoder: the round-trip check must not lean on code64().
|
||||
*/
|
||||
static unsigned char *sf_unb64(const char *s, size_t len, size_t *outlen) {
|
||||
static const char alpha[] =
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
unsigned char *out = (unsigned char *) malloct(len / 4 * 3 + 4);
|
||||
unsigned int acc = 0;
|
||||
size_t i, n = 0;
|
||||
int bits = 0;
|
||||
|
||||
if (out == NULL)
|
||||
return NULL;
|
||||
for (i = 0; i < len; i++) {
|
||||
const char *const p = s[i] != '\0' ? strchr(alpha, s[i]) : NULL;
|
||||
|
||||
if (s[i] == '=')
|
||||
break;
|
||||
if (p == NULL) {
|
||||
freet(out);
|
||||
return NULL;
|
||||
}
|
||||
acc = (acc << 6) | (unsigned int) (p - alpha);
|
||||
bits += 6;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out[n++] = (unsigned char) ((acc >> bits) & 0xff);
|
||||
}
|
||||
}
|
||||
*outlen = n;
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Decoded payload of the first data: URI with that MIME, as a NUL-terminated
|
||||
buffer the caller freet()s. NULL when absent or undecodable. */
|
||||
static char *sf_decode(const char *hay, const char *mime, size_t *outlen) {
|
||||
char prefix[128];
|
||||
size_t len = 0, dlen = 0;
|
||||
const char *b64;
|
||||
unsigned char *raw;
|
||||
|
||||
snprintf(prefix, sizeof(prefix), "data:%s;base64,", mime);
|
||||
b64 = sf_payload(hay, prefix, &len);
|
||||
if (b64 == NULL)
|
||||
return NULL;
|
||||
raw = sf_unb64(b64, len, &dlen);
|
||||
if (raw == NULL)
|
||||
return NULL;
|
||||
raw[dlen] = '\0';
|
||||
if (outlen != NULL)
|
||||
*outlen = dlen;
|
||||
return (char *) raw;
|
||||
}
|
||||
|
||||
/* How many data: URIs of that MIME nest inside each other, starting at hay. */
|
||||
static int sf_nesting(const char *hay, const char *mime) {
|
||||
char *cur = strdupt(hay);
|
||||
int n = 0;
|
||||
|
||||
while (cur != NULL) {
|
||||
char *const inner = sf_decode(cur, mime, NULL);
|
||||
|
||||
freet(cur);
|
||||
cur = inner;
|
||||
if (inner != NULL)
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Above the tag parser's attribute limit, so the fixture crosses it. */
|
||||
#define SF_ST_MAX_ATTRS 64
|
||||
|
||||
/* The 12-byte asset: high bytes and an embedded NUL, so a text-shaped copy
|
||||
would be caught. */
|
||||
static const char sf_png[] = "\x89PNG\r\n\x1a\n\x00\x01\x02\xff";
|
||||
#define SF_PNG_LEN 12
|
||||
|
||||
static const char sf_page[] =
|
||||
"<html><head>\n"
|
||||
"<link rel=\"stylesheet\" href=\"css/main.css\">\n"
|
||||
"<link rel=\"canonical\" href=\"other.html\">\n"
|
||||
"<title>t</title>\n"
|
||||
"<style>body { background: url(\"img/a%20b.png\"); }</style>\n"
|
||||
"</head><body>\n"
|
||||
"<img src=\"img/a%20b.png\" srcset=\"img/a%20b.png 1x, img/big.png 2x\">\n"
|
||||
"<link rel=\"icon\" href=\"icon.png\">\n"
|
||||
"<link rel=\"preload\" as=\"font\" href=\"font/f.woff2\">\n"
|
||||
"<img src=\"data:image/gif;base64,QUJD\">\n"
|
||||
/* Each has a real file where its guard's removal would land it; without
|
||||
that they stay links either way, the target merely being absent. */
|
||||
"<img src=\"http://example.com/x.png\">\n"
|
||||
"<img src=\"//example.com/x.png\">\n"
|
||||
"<input type=\"image\" src=\"img/in.png\">\n"
|
||||
/* Lazy loading: src is the placeholder, the real image rides data-src. */
|
||||
"<img src=\"img/ph.png\" data-src=\"img/lz.png\" "
|
||||
"data-srcset=\"img/lz2.png 2x\" lowsrc=\"img/low.png\">\n"
|
||||
"<object data=\"img/ob.png\"></object>\n"
|
||||
"<embed src=\"img/em.png\">\n"
|
||||
"<img data-src=\"other.html\">\n"
|
||||
/* What a first pass emits: re-resolving it is what a second pass must not
|
||||
do, and the fallback type would inline whatever the walk found. */
|
||||
"<link rel=\"stylesheet\" href=\"data:text/css;base64,QUJD\">\n"
|
||||
"<video poster=\"img/po.png\" controls>"
|
||||
"<source src=\"v.mp4\" type=\"video/mp4\"></video>\n"
|
||||
"<svg><image href=\"img/sv.png\"/></svg>\n"
|
||||
"<table background=\"img/bg.png\"><tr><td>x</td></tr></table>\n"
|
||||
/* The second is what bites: drop the clamp and its leading ".." lands it
|
||||
back on <root>/img/a b.png. The first can only 404 either way. */
|
||||
"<img src=\"../escape.png\">\n"
|
||||
"<img src=\"../img/a%20b.png\">\n"
|
||||
"<a href=\"img/a%20b.png\">link</a>\n"
|
||||
"<script src=\"js/app.js\"></script>\n"
|
||||
"<script>var s = \"</scripting>\"; var t = \"<img src='img/a%20b.png'>\";"
|
||||
"</script>\n"
|
||||
"<img src=\"missing.png\" >\n"
|
||||
"<!--><img src=\"img/a%20b.png\">\n"
|
||||
"<div style=\"background:url(img/a%20b.png)\"></div>\n"
|
||||
"<div style='content:\"x\"; background:url(img/a%20b.png)'></div>\n"
|
||||
"</body></html>\n";
|
||||
|
||||
/* Lay a small mirror down under root. */
|
||||
static void sf_fixture(const char *root) {
|
||||
/* The over-cap url() is what drives the rebase fallback: a reference an
|
||||
inlined stylesheet could not embed has to come out relative to the page,
|
||||
not to the stylesheet, or it dangles. */
|
||||
static const char css[] =
|
||||
"@import \"sub/nested.css\";\n"
|
||||
"@import url(\"sub/two.css\");\n"
|
||||
"@import \"a\\\"url(../img/a b.png)b.css\";\n"
|
||||
"@font-face { font-family: f; src: url(../font/f.woff2); }\n"
|
||||
"body { background: url(../img/a b.png); }\n"
|
||||
"div { background: url(../img/big.png); }\n"
|
||||
"/* url(../img/never.png) */\n";
|
||||
static const char nested[] = "div { background: url(../../img/a b.png); }\n";
|
||||
static const char two[] = "p { background: url(../../img/a b.png); }\n";
|
||||
static const char deep[] =
|
||||
"<html><head><link rel=\"stylesheet\" href=\"../../css/main.css\">\n"
|
||||
"</head><body>d</body></html>\n";
|
||||
static const char js[] = "var app = 1;\n";
|
||||
char big[4096];
|
||||
|
||||
memset(big, 'B', sizeof(big));
|
||||
sf_put(root, "page.html", sf_page, sizeof(sf_page) - 1);
|
||||
sf_put(root, "deep/sub/page.html", deep, sizeof(deep) - 1);
|
||||
sf_put(root, "other.html", "<html>o</html>", 14);
|
||||
sf_put(root, "css/main.css", css, sizeof(css) - 1);
|
||||
sf_put(root, "css/sub/nested.css", nested, sizeof(nested) - 1);
|
||||
sf_put(root, "css/sub/two.css", two, sizeof(two) - 1);
|
||||
sf_put(root, "js/app.js", js, sizeof(js) - 1);
|
||||
sf_put(root, "img/a b.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/big.png", big, sizeof(big));
|
||||
sf_put(root, "img/in.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/po.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/sv.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/bg.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/ph.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/lz.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/lz2.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/low.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/ob.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/em.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "icon.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "font/f.woff2", "wOF2\x00\x01", 6);
|
||||
/* Where a guard's removal would land each reference that has to stay a
|
||||
link. The colon-bearing two are impossible on Windows, where the scheme
|
||||
guard then only gets the weaker "the link survived" check. */
|
||||
sf_put(root, "img/never.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "example.com/x.png", sf_png, SF_PNG_LEN);
|
||||
sf_colon_ok =
|
||||
sf_try_put(root, "http:/example.com/x.png", sf_png, SF_PNG_LEN) &&
|
||||
sf_try_put(root, "data:text/css;base64,QUJD", "p{}", 3);
|
||||
{ /* More attributes than the tag parser records, with a '>' inside a quoted
|
||||
value: the give-up path must not rescan quote-blind. */
|
||||
String wide = STRING_EMPTY;
|
||||
int n;
|
||||
|
||||
StringCopy(wide, "<html><body><p");
|
||||
for (n = 0; n <= SF_ST_MAX_ATTRS; n++) {
|
||||
char one[32];
|
||||
|
||||
assertf(sprintfbuff(one, " a%d=1", n));
|
||||
StringCat(wide, one);
|
||||
}
|
||||
StringCat(wide,
|
||||
" title=\"> <img src=img/a%20b.png> \">end</p></body></html>");
|
||||
sf_put(root, "wide.html", StringBuff(wide), StringLength(wide));
|
||||
StringFree(wide);
|
||||
}
|
||||
sf_put(root, "v.mp4",
|
||||
"\x00\x00\x00\x18"
|
||||
"ftypisom",
|
||||
12);
|
||||
}
|
||||
|
||||
/* -#test=singlefile <dir>: rewrite a hand-built mirror and check what gets
|
||||
inlined, what must keep its link, the per-asset cap, and idempotence. */
|
||||
static int st_singlefile(httrackp *opt, int argc, char **argv) {
|
||||
char BIGSTK root[HTS_URLMAXSIZE];
|
||||
char BIGSTK page[HTS_URLMAXSIZE * 2];
|
||||
const LLint saved_cap = opt->single_file_max_size;
|
||||
char *out, *css, *nested;
|
||||
size_t outlen = 0, len = 0;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "singlefile: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
sf_err = 0;
|
||||
sf_put(argv[0], "escape.png", sf_png,
|
||||
SF_PNG_LEN); /* just outside the mirror */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror/");
|
||||
sf_fixture(root);
|
||||
fconcat(page, sizeof(page), root, "page.html");
|
||||
|
||||
/* Cap between the small assets and big.png. */
|
||||
opt->single_file_max_size = 1024;
|
||||
sf_check(singlefile_rewrite_file(opt, root, page),
|
||||
"first pass changed nothing");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
|
||||
/* Inlined, and the payload is the file's exact bytes. */
|
||||
{
|
||||
char *img = sf_decode(out, "image/png", &len);
|
||||
|
||||
sf_check(img != NULL && len == SF_PNG_LEN &&
|
||||
memcmp(img, sf_png, SF_PNG_LEN) == 0,
|
||||
"image payload does not round-trip");
|
||||
freet(img);
|
||||
}
|
||||
{
|
||||
char *js = sf_decode(out, "application/x-javascript", &len);
|
||||
|
||||
sf_check(js != NULL && len == 13 && memcmp(js, "var app = 1;\n", 13) == 0,
|
||||
"script payload does not round-trip");
|
||||
freet(js);
|
||||
}
|
||||
sf_check(strstr(out, "href=\"data:text/css;base64,") != NULL,
|
||||
"stylesheet not inlined into the link");
|
||||
{
|
||||
char *font = sf_decode(out, "font/woff2", &len);
|
||||
|
||||
sf_check(font != NULL && len == 6 && memcmp(font, "wOF2\x00\x01", 6) == 0,
|
||||
"rel=preload font payload does not round-trip");
|
||||
freet(font);
|
||||
}
|
||||
|
||||
/* Every other (tag, attribute) rule in the table. */
|
||||
sf_check(strstr(out, "icon.png") == NULL, "rel=icon not inlined");
|
||||
sf_check(strstr(out, "img/in.png") == NULL, "input src not inlined");
|
||||
sf_check(strstr(out, "img/po.png") == NULL, "video poster not inlined");
|
||||
sf_check(strstr(out, "img/sv.png") == NULL, "svg image href not inlined");
|
||||
sf_check(strstr(out, "img/bg.png") == NULL,
|
||||
"legacy background attribute not inlined");
|
||||
sf_check(strstr(out, "img/ob.png") == NULL, "object data not inlined");
|
||||
sf_check(strstr(out, "img/em.png") == NULL, "embed src not inlined");
|
||||
sf_check(strstr(out, "img/ph.png") == NULL, "lazy placeholder not inlined");
|
||||
sf_check(strstr(out, "img/lz.png") == NULL, "data-src not inlined");
|
||||
sf_check(strstr(out, "img/lz2.png") == NULL, "data-srcset not inlined");
|
||||
sf_check(strstr(out, "img/low.png") == NULL, "lowsrc not inlined");
|
||||
/* The class gate, not the attribute name, is what keeps a page out. */
|
||||
sf_check(strstr(out, "data-src=\"other.html\"") != NULL,
|
||||
"data-src pointing at a page was inlined");
|
||||
|
||||
sf_check(strstr(out, "<a href=\"img/a%20b.png\">") != NULL, "anchor inlined");
|
||||
sf_check(strstr(out, "href=\"other.html\"") != NULL, "rel=canonical inlined");
|
||||
sf_check(strstr(out, "src=\"v.mp4\"") != NULL, "video source inlined");
|
||||
sf_check(strstr(out, "src=\"http://example.com/x.png\"") != NULL,
|
||||
"absolute URL rewritten");
|
||||
sf_check(strstr(out, "src=\"//example.com/x.png\"") != NULL,
|
||||
"site-root-relative URL rewritten");
|
||||
sf_check(sf_count(out, "QUJD") == 2, "existing data: URI not preserved");
|
||||
sf_check(strstr(out, "src=\"../escape.png\"") != NULL,
|
||||
"a reference outside the mirror was resolved");
|
||||
sf_check(strstr(out, "src=\"../img/a%20b.png\"") != NULL,
|
||||
"a leading .. was dropped instead of rejected");
|
||||
sf_check(strstr(out, "var t = \"<img src='img/a%20b.png'>\";") != NULL,
|
||||
"script body rewritten past a </scripting> lookalike");
|
||||
|
||||
/* Nothing an attribute value cannot hold: url() stays unquoted, and a quote
|
||||
that was already in the CSS is escaped. */
|
||||
sf_check(strstr(out, "url(\"data:") == NULL,
|
||||
"a quoted url() would end a style attribute");
|
||||
sf_check(strstr(out, "style=\"content:"x"; background:url(data:") !=
|
||||
NULL,
|
||||
"quote inside a rewritten style attribute not escaped");
|
||||
|
||||
sf_check(strstr(out, "img/big.png 2x") != NULL, "over-cap asset inlined");
|
||||
sf_check(strstr(out, " 1x") != NULL, "srcset descriptor lost");
|
||||
sf_check(sf_count(out, "img/a%20b.png") ==
|
||||
3, /* the anchor, the script body, and the ".." one */
|
||||
"an inlinable reference was left as a link");
|
||||
|
||||
/* The inlined stylesheet carries its own @import and url() inlined. */
|
||||
css = sf_decode(out, "text/css", NULL);
|
||||
sf_check(css != NULL, "stylesheet payload undecodable");
|
||||
if (css != NULL) {
|
||||
sf_check(strstr(css, "@import \"data:text/css;base64,") != NULL,
|
||||
"@import not inlined");
|
||||
sf_check(strstr(css, "url(data:image/png;base64,") != NULL,
|
||||
"url() in stylesheet not inlined");
|
||||
sf_check(strstr(css, "url(../img/never.png)") != NULL,
|
||||
"url() inside a CSS comment was rewritten");
|
||||
sf_check(strstr(css, "url(data:font/woff2;base64,") != NULL,
|
||||
"@font-face src not inlined");
|
||||
sf_check(strstr(css, "@import url(data:text/css;base64,") != NULL,
|
||||
"@import url() form not inlined");
|
||||
sf_check(strstr(css, "url(../img/a b.png)b.css") != NULL,
|
||||
"url() inside a string with an escaped quote was rewritten");
|
||||
/* The over-cap url() read ../img/big.png from css/; this page sits at the
|
||||
root, so it has to come back out as img/big.png or it dangles. */
|
||||
sf_check(strstr(css, "url(img/big.png)") != NULL,
|
||||
"over-cap url() not rebased onto the page's directory");
|
||||
nested = sf_decode(css, "text/css", NULL);
|
||||
sf_check(nested != NULL &&
|
||||
strstr(nested, "url(data:image/png;base64,") != NULL,
|
||||
"url() in the @import'ed stylesheet not inlined");
|
||||
freet(nested);
|
||||
freet(css);
|
||||
}
|
||||
|
||||
/* A tag with more attributes than the parser records comes back byte for
|
||||
byte, quoted '>' and all, instead of being re-scanned as markup. */
|
||||
{
|
||||
char BIGSTK wide[HTS_URLMAXSIZE * 2];
|
||||
char *before, *after;
|
||||
|
||||
fconcat(wide, sizeof(wide), root, "wide.html");
|
||||
before = readfile_utf8(wide);
|
||||
assertf(before != NULL);
|
||||
(void) singlefile_rewrite_file(opt, root, wide);
|
||||
after = readfile_utf8(wide);
|
||||
sf_check(after != NULL && strcmp(before, after) == 0,
|
||||
"an over-wide tag was rewritten");
|
||||
freet(after);
|
||||
freet(before);
|
||||
}
|
||||
|
||||
/* The same stylesheet from two directories down, where the rebase has to
|
||||
climb: the emitted path must stay inside the mirror and name the file. */
|
||||
{
|
||||
char BIGSTK deep[HTS_URLMAXSIZE * 2];
|
||||
char *dout, *dcss;
|
||||
|
||||
fconcat(deep, sizeof(deep), root, "deep/sub/page.html");
|
||||
sf_check(singlefile_rewrite_file(opt, root, deep),
|
||||
"deep page not rewritten");
|
||||
dout = readfile_utf8(deep);
|
||||
assertf(dout != NULL);
|
||||
dcss = sf_decode(dout, "text/css", NULL);
|
||||
sf_check(dcss != NULL && strstr(dcss, "url(../../img/big.png)") != NULL,
|
||||
"over-cap url() not rebased from a nested page");
|
||||
freet(dcss);
|
||||
freet(dout);
|
||||
}
|
||||
|
||||
/* Idempotence: a second pass must find nothing and leave the bytes alone. */
|
||||
sf_check(!singlefile_rewrite_file(opt, root, page), "second pass rewrote");
|
||||
{
|
||||
char *again = readfile_utf8(page);
|
||||
|
||||
sf_check(again != NULL && strcmp(again, out) == 0,
|
||||
"second pass changed the page");
|
||||
freet(again);
|
||||
}
|
||||
freet(out);
|
||||
|
||||
/* Same page, a cap above big.png: it now inlines. */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror2/");
|
||||
sf_fixture(root);
|
||||
fconcat(page, sizeof(page), root, "page.html");
|
||||
opt->single_file_max_size = 1024 * 1024;
|
||||
sf_check(singlefile_rewrite_file(opt, root, page),
|
||||
"raised-cap pass changed nothing");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
sf_check(strstr(out, "img/big.png 2x") == NULL,
|
||||
"asset under the raised cap still a link");
|
||||
freet(out);
|
||||
|
||||
/* Nothing inlines, so the rewriter must be byte transparent. Assert on its
|
||||
output: the file it declined to write could not have changed regardless. */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror3/");
|
||||
sf_fixture(root);
|
||||
fconcat(page, sizeof(page), root, "page.html");
|
||||
opt->single_file_max_size = 1;
|
||||
sf_check(!singlefile_rewrite_file(opt, root, page), "one-byte cap inlined");
|
||||
{
|
||||
String verbatim = STRING_EMPTY;
|
||||
|
||||
StringClear(verbatim);
|
||||
(void) singlefile_rewrite_html(opt, root, page, sf_page,
|
||||
sizeof(sf_page) - 1,
|
||||
SINGLEFILE_MAX_PAGE_SIZE, &verbatim);
|
||||
sf_check(StringLength(verbatim) == sizeof(sf_page) - 1 &&
|
||||
memcmp(StringBuff(verbatim), sf_page, sizeof(sf_page) - 1) ==
|
||||
0,
|
||||
"a page with nothing to inline was re-serialized differently");
|
||||
StringFree(verbatim);
|
||||
}
|
||||
(void) outlen;
|
||||
|
||||
/* The per-page budget against a self-importing stylesheet. The large-budget
|
||||
run is the control: it proves the fan-out is real, so the small one was
|
||||
cut short by the budget and not by the fixture. */
|
||||
{
|
||||
static const char bomb_css[] = "@import \"b.css\";@import \"b.css\";"
|
||||
"@import \"b.css\";@import \"b.css\";\n";
|
||||
static const char bomb_html[] = "<html><head>"
|
||||
"<link rel=\"stylesheet\" href=\"b.css\">"
|
||||
"</head></html>\n";
|
||||
const size_t css_len = sizeof(bomb_css) - 1;
|
||||
String small = STRING_EMPTY, large = STRING_EMPTY;
|
||||
|
||||
fconcat(root, sizeof(root), argv[0], "bomb/");
|
||||
sf_put(root, "b.css", bomb_css, css_len);
|
||||
fconcat(page, sizeof(page), root, "page.html");
|
||||
opt->single_file_max_size = 1024 * 1024;
|
||||
StringClear(small);
|
||||
StringClear(large);
|
||||
(void) singlefile_rewrite_html(opt, root, page, bomb_html,
|
||||
sizeof(bomb_html) - 1, (LLint) css_len * 3,
|
||||
&small);
|
||||
(void) singlefile_rewrite_html(opt, root, page, bomb_html,
|
||||
sizeof(bomb_html) - 1,
|
||||
SINGLEFILE_MAX_PAGE_SIZE, &large);
|
||||
sf_check(StringLength(large) > 4096, "the @import bomb did not fan out");
|
||||
sf_check(StringLength(small) < StringLength(large) / 8,
|
||||
"the per-page budget did not cut the fan-out short");
|
||||
/* Three files fit in three file-lengths only if the budget is charged
|
||||
before each nested rewrite; charging after measured five levels. */
|
||||
sf_check(sf_nesting(StringBuff(small), "text/css") == 3,
|
||||
"the per-page budget was not charged as each asset was taken");
|
||||
StringFree(small);
|
||||
StringFree(large);
|
||||
}
|
||||
|
||||
if (!sf_colon_ok)
|
||||
printf("singlefile: ':' is not a legal filename here, so the scheme guard "
|
||||
"only gets the weaker check\n");
|
||||
opt->single_file_max_size = saved_cap;
|
||||
printf("singlefile: %s\n", sf_err ? "FAIL" : "OK");
|
||||
return sf_err;
|
||||
}
|
||||
|
||||
// -#test=longpath <dir>: round-trip a >MAX_PATH (260) file through the file
|
||||
// wrappers, exercising hts_pathToUCS2's \\?\ prefixing on Windows (#133).
|
||||
static int st_longpath(httrackp *opt, int argc, char **argv) {
|
||||
@@ -5580,6 +6109,9 @@ static const struct selftest_entry {
|
||||
{"warc-wacz", "<dir>", "--wacz package: layout, STORE mode, sha256 digests",
|
||||
st_warc_wacz},
|
||||
#endif
|
||||
{"singlefile", "<dir>",
|
||||
"--single-file: what is inlined, the per-asset cap, idempotence",
|
||||
st_singlefile},
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
1160
src/htssinglefile.c
Normal file
1160
src/htssinglefile.c
Normal file
File diff suppressed because it is too large
Load Diff
84
src/htssinglefile.h
Normal file
84
src/htssinglefile.h
Normal file
@@ -0,0 +1,84 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* --single-file: post-mirror pass embedding each page's assets as data: URIs.
|
||||
Internal, not installed. Unlike -%M (one MHT archive for the whole mirror)
|
||||
it rewrites the saved pages in place and keeps page-to-page links relative,
|
||||
so the mirror stays browsable and every page also stands alone.
|
||||
Limitation: an inlined stylesheet becomes a data: URL, whose path is opaque,
|
||||
so an asset inside it that stayed a link (over the cap) no longer resolves.
|
||||
Raise --single-file-max-size past it to inline it too. */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_SINGLEFILE_DEFH
|
||||
#define HTS_SINGLEFILE_DEFH
|
||||
|
||||
#include "htsopt.h"
|
||||
#include "htsstrings.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* --single-file-max-size default: a bigger asset stays an ordinary link. */
|
||||
#define SINGLEFILE_DEFAULT_MAX_SIZE (10 * 1024 * 1024)
|
||||
|
||||
/* Never base64 more than this, whatever the cap: code64() sizes in int. */
|
||||
#define SINGLEFILE_HARD_MAX_SIZE (256 * 1024 * 1024)
|
||||
|
||||
/* Total bytes one page may inline. Nested @import fans out multiplicatively,
|
||||
so a few hundred bytes of hostile CSS can otherwise ask for gigabytes. */
|
||||
#define SINGLEFILE_MAX_PAGE_SIZE (64 * 1024 * 1024)
|
||||
|
||||
/* Rewrite every HTML page the mirror produced. No-op unless opt->single_file;
|
||||
call once the tree is final, after the update purge. */
|
||||
void singlefile_process_mirror(httrackp *opt);
|
||||
|
||||
/* Rewrite one HTML document held in memory, appending the result to out.
|
||||
root is the mirror directory that references may not escape; page_path is
|
||||
the document's own path under it (both UTF-8, '/' or native separators).
|
||||
page_budget caps the total inlined bytes, since nested @import fans out
|
||||
multiplicatively; the mirror pass passes SINGLEFILE_MAX_PAGE_SIZE.
|
||||
Returns HTS_TRUE if at least one reference was replaced; out may still
|
||||
differ from the input when that is HTS_FALSE, since a style or srcset value
|
||||
is re-serialized in place. */
|
||||
hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
|
||||
const char *page_path, const char *html,
|
||||
size_t html_len, LLint page_budget,
|
||||
String *out);
|
||||
|
||||
/* singlefile_rewrite_html() over the file at page_path, rewritten in place.
|
||||
Returns HTS_TRUE if the file changed. */
|
||||
hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
|
||||
const char *page_path);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -131,6 +131,7 @@
|
||||
<ClCompile Include="htsproxy.c" />
|
||||
<ClCompile Include="htsrobots.c" />
|
||||
<ClCompile Include="htsselftest.c" />
|
||||
<ClCompile Include="htssinglefile.c" />
|
||||
<ClCompile Include="htssniff.c" />
|
||||
<ClCompile Include="htsthread.c" />
|
||||
<ClCompile Include="htstools.c" />
|
||||
|
||||
@@ -156,6 +156,7 @@ BOXES = [
|
||||
("index", "Index", None, None),
|
||||
("index2", "WordIndex", "--search-index", None),
|
||||
("ftpprox", "UseHTTPProxyForFTP", "--httpproxy-ftp", None),
|
||||
("singlefile", "SingleFile", "--single-file", None),
|
||||
]
|
||||
|
||||
for field, key, when_set, when_clear in BOXES:
|
||||
|
||||
379
tests/94_local-single-file.test
Normal file
379
tests/94_local-single-file.test
Normal file
@@ -0,0 +1,379 @@
|
||||
#!/bin/bash
|
||||
# Issue #713: --single-file inlines each page's assets as data: URIs. End to
|
||||
# end over a real crawl, decoding the payloads against what the server served.
|
||||
set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_713.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
htssrv=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
|
||||
test -z "$htssrv" || kill -9 "$htssrv" 2>/dev/null || true
|
||||
wait "$htssrv" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
htssrv=
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# --- the site ---------------------------------------------------------------
|
||||
doc="${tmpdir}/doc"
|
||||
mkdir -p "$doc"
|
||||
cat >"${doc}/index.html" <<'EOF'
|
||||
<html><head>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<link rel="canonical" href="other.html">
|
||||
</head><body>
|
||||
<img src="pixel.svg" alt="p">
|
||||
<img src="big.svg" alt="b">
|
||||
<img src="pixel.svg" data-src="lazy.svg" alt="l">
|
||||
<script src="app.js"></script>
|
||||
<a href="other.html">other</a>
|
||||
</body></html>
|
||||
EOF
|
||||
cat >"${doc}/other.html" <<'EOF'
|
||||
<html><body><a href="index.html">back</a></body></html>
|
||||
EOF
|
||||
cat >"${doc}/style.css" <<'EOF'
|
||||
@import "nested.css";
|
||||
body { background: url(pixel.svg); }
|
||||
EOF
|
||||
cat >"${doc}/nested.css" <<'EOF'
|
||||
div { background: url(pixel.svg); }
|
||||
EOF
|
||||
printf 'var app = 1;\n' >"${doc}/app.js"
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><rect width="1" height="1"/></svg>\n' \
|
||||
>"${doc}/pixel.svg"
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><circle r="1"/></svg>\n' \
|
||||
>"${doc}/lazy.svg"
|
||||
{
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><!--'
|
||||
head -c 20000 /dev/zero | tr '\0' 'x'
|
||||
printf '%s\n' '--><rect width="1" height="1"/></svg>'
|
||||
} >"${doc}/big.svg"
|
||||
|
||||
# --- server -----------------------------------------------------------------
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
"$python" "$server" --root "$(nativepath "$doc")" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- crawl ------------------------------------------------------------------
|
||||
# The cap sits between pixel.svg and big.svg, so one inlines and one must not.
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=0)
|
||||
httrack "${common[@]}" --single-file --single-file-max-size 4096 \
|
||||
"${base}/index.html" >"${tmpdir}/log1" 2>&1
|
||||
|
||||
page="${out}/127.0.0.1_${port}/index.html"
|
||||
test -f "$page" || {
|
||||
echo "FAIL: no mirrored page at $page"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- what must be inlined, byte for byte ------------------------------------
|
||||
# The decoder is python, not httrack, so a broken encoder cannot agree with
|
||||
# itself here.
|
||||
extract() {
|
||||
"$python" - "$1" "$2" "$3" <<'PY'
|
||||
import base64, re, sys
|
||||
html = open(sys.argv[1], "rb").read().decode("utf-8", "replace")
|
||||
m = re.search(r'data:%s;base64,([A-Za-z0-9+/=]*)' % re.escape(sys.argv[2]), html)
|
||||
if m is None:
|
||||
sys.exit(1)
|
||||
open(sys.argv[3], "wb").write(base64.b64decode(m.group(1)))
|
||||
PY
|
||||
}
|
||||
|
||||
printf '[image inlined and identical] ..\t'
|
||||
extract "$page" "image/svg+xml" "${tmpdir}/got.svg" || {
|
||||
echo "FAIL: no inlined image"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/pixel.svg" "${tmpdir}/got.svg" || {
|
||||
echo "FAIL: inlined image differs from the served file"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[script inlined and identical] ..\t'
|
||||
extract "$page" "application/x-javascript" "${tmpdir}/got.js" || {
|
||||
echo "FAIL: no inlined script"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/app.js" "${tmpdir}/got.js" || {
|
||||
echo "FAIL: inlined script differs from the served file"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[lazy-loaded image inlined] ..\t'
|
||||
# src is the placeholder a real page ships; the asset rides data-src.
|
||||
grep -q 'data-src="data:image/svg+xml;base64,' "$page" || {
|
||||
echo "FAIL: data-src was not inlined"
|
||||
exit 1
|
||||
}
|
||||
if grep -q 'data-src="lazy.svg"' "$page"; then
|
||||
echo "FAIL: data-src kept its link"
|
||||
exit 1
|
||||
fi
|
||||
test -f "${out}/127.0.0.1_${port}/lazy.svg" || {
|
||||
echo "FAIL: the lazy image was never downloaded, so nothing was proven"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[stylesheet inlined, recursively] ..\t'
|
||||
extract "$page" "text/css" "${tmpdir}/got.css" || {
|
||||
echo "FAIL: no inlined stylesheet"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'url(data:image/svg+xml;base64,' "${tmpdir}/got.css" || {
|
||||
echo "FAIL: url() inside the inlined stylesheet was not inlined"
|
||||
exit 1
|
||||
}
|
||||
extract "${tmpdir}/got.css" "text/css" "${tmpdir}/got2.css" || {
|
||||
echo "FAIL: @import was not inlined"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'url(data:image/svg+xml;base64,' "${tmpdir}/got2.css" || {
|
||||
echo "FAIL: url() inside the @import'ed stylesheet was not inlined"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- what must keep its link ------------------------------------------------
|
||||
printf '[page links stay relative] ..\t'
|
||||
grep -q 'href="other.html"' "$page" || {
|
||||
echo "FAIL: the link to another page was not left relative"
|
||||
exit 1
|
||||
}
|
||||
test -f "${out}/127.0.0.1_${port}/other.html" || {
|
||||
echo "FAIL: the linked page is missing from the mirror"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[over-cap asset stays a link] ..\t'
|
||||
grep -q 'src="big.svg"' "$page" || {
|
||||
echo "FAIL: the over-cap asset did not keep its link"
|
||||
exit 1
|
||||
}
|
||||
test -f "${out}/127.0.0.1_${port}/big.svg" || {
|
||||
echo "FAIL: the over-cap asset is missing from the mirror"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'over the 4096-byte cap' "${out}/hts-log.txt" || {
|
||||
echo "FAIL: the over-cap asset was not reported in the log"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[the mirror still works] ..\t'
|
||||
if grep -q 'href="style.css"' "$page"; then
|
||||
echo "FAIL: the stylesheet link survived"
|
||||
exit 1
|
||||
fi
|
||||
test -f "${out}/127.0.0.1_${port}/style.css" || {
|
||||
echo "FAIL: the mirror lost the standalone stylesheet"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- a second run must not re-encode ----------------------------------------
|
||||
printf '[idempotent under --update] ..\t'
|
||||
# HTTrack stamps the time into its own footer comment, so compare without it.
|
||||
grep -v 'Mirrored from' "$page" >"${tmpdir}/page1.html"
|
||||
httrack "${common[@]}" --single-file --single-file-max-size 4096 --update \
|
||||
"${base}/index.html" >"${tmpdir}/log2" 2>&1
|
||||
grep -v 'Mirrored from' "$page" >"${tmpdir}/page2.html"
|
||||
cmp -s "${tmpdir}/page1.html" "${tmpdir}/page2.html" || {
|
||||
echo "FAIL: the second run changed the page"
|
||||
exit 1
|
||||
}
|
||||
# A double-encoded payload would decode to another data: URI, not to the SVG.
|
||||
extract "$page" "image/svg+xml" "${tmpdir}/got2.svg" || {
|
||||
echo "FAIL: no inlined image after the second run"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/pixel.svg" "${tmpdir}/got2.svg" || {
|
||||
echo "FAIL: the second run re-encoded the inlined image"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[-%%Z0 and a bad cap] ..\t'
|
||||
# -%Z0 turns it back off, so the page must come out exactly as the crawl left
|
||||
# it; a non-numeric cap is rejected and the built-in default used instead.
|
||||
off="${tmpdir}/off"
|
||||
mkdir "$off"
|
||||
httrack -O "$off" --quiet --disable-security-limits --robots=0 --timeout=30 \
|
||||
--retries=0 -%Z0 "${base}/index.html" >"${tmpdir}/log3" 2>&1
|
||||
offpage="${off}/127.0.0.1_${port}/index.html"
|
||||
if grep -q 'base64,' "$offpage"; then
|
||||
echo "FAIL: -%Z0 still inlined"
|
||||
exit 1
|
||||
fi
|
||||
bad="${tmpdir}/bad"
|
||||
mkdir "$bad"
|
||||
httrack -O "$bad" --quiet --disable-security-limits --robots=0 --timeout=30 \
|
||||
--retries=0 --single-file-max-size notanumber "${base}/index.html" \
|
||||
>"${tmpdir}/log4" 2>&1
|
||||
badpage="${bad}/127.0.0.1_${port}/index.html"
|
||||
# The default cap is 10 MB, so everything here inlines, big.svg included.
|
||||
grep -q 'data:image/svg+xml;base64,' "$badpage" || {
|
||||
echo "FAIL: a rejected cap did not fall back to the default"
|
||||
exit 1
|
||||
}
|
||||
if grep -q 'src="big.svg"' "$badpage"; then
|
||||
echo "FAIL: the default cap did not cover big.svg"
|
||||
exit 1
|
||||
fi
|
||||
echo OK
|
||||
|
||||
printf '[the rewritten page keeps the mirror file mode] ..\t'
|
||||
# The rewrite spools to a temp and renames, bypassing the engine's chmod.
|
||||
if is_windows; then
|
||||
echo "SKIP (no POSIX file modes)"
|
||||
else
|
||||
umaskdir="${tmpdir}/umask"
|
||||
mkdir "$umaskdir"
|
||||
(
|
||||
umask 077
|
||||
httrack -O "$umaskdir" --quiet --disable-security-limits --robots=0 \
|
||||
--timeout=30 --retries=0 --single-file "${base}/index.html" \
|
||||
>"${tmpdir}/log5" 2>&1
|
||||
)
|
||||
umaskroot="${umaskdir}/127.0.0.1_${port}"
|
||||
mode() { "$python" -c 'import os,sys
|
||||
print("%04o" % (os.stat(sys.argv[1]).st_mode & 0o777))' "$1"; }
|
||||
pagemode=$(mode "${umaskroot}/index.html")
|
||||
assetmode=$(mode "${umaskroot}/big.svg")
|
||||
test "$pagemode" = "$assetmode" || {
|
||||
echo "FAIL: page is $pagemode, the asset beside it is $assetmode"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
fi
|
||||
|
||||
# --- the rewriter in isolation, over a hand-built mirror --------------------
|
||||
# After the crawl assertions, so a failure here cannot hide them: the two cover
|
||||
# different ground (this one reaches media, anchors and the mirror-root clamp).
|
||||
printf '[engine self-test] ..\t'
|
||||
st=$(httrack -O /dev/null "-#test=singlefile" "${tmpdir}/st/" 2>/dev/null)
|
||||
case "$st" in
|
||||
*": OK") echo OK ;;
|
||||
*)
|
||||
echo "FAIL: $st"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- the web GUI must emit the same options ---------------------------------
|
||||
distdir=$(cd "${top_srcdir}" && pwd)
|
||||
printf '[project reload maps the keys back] ..\t'
|
||||
grep -q 'do:copy:SingleFile:singlefile' "${distdir}/html/server/step2.html" || {
|
||||
echo "FAIL: step2.html does not restore SingleFile"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'do:copy:SingleFileMaxSize:singlefilemax' "${distdir}/html/server/step2.html" || {
|
||||
echo "FAIL: step2.html does not restore SingleFileMaxSize"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[webhttrack renders the options] ..\t'
|
||||
# The Windows job builds httrack.exe only and reaches this file through its
|
||||
# *_local-*.test glob, so report a skip there rather than a pass.
|
||||
if ! command -v htsserver >/dev/null; then
|
||||
echo "SKIP (no htsserver in PATH)"
|
||||
exit 77
|
||||
fi
|
||||
sport=$("$python" -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()')
|
||||
srvlog="${tmpdir}/htsserver.log"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
export HOME="${tmpdir}/home"
|
||||
mkdir -p "$HOME"
|
||||
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
|
||||
) &
|
||||
htssrv=$!
|
||||
url=
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "$srvlog" 2>/dev/null) && test -n "$url" && break
|
||||
kill -0 "$htssrv" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || {
|
||||
echo "FAIL: htsserver did not start: $(cat "$srvlog")"
|
||||
exit 1
|
||||
}
|
||||
"$python" - "$url" <<'PY' || exit 1
|
||||
import re, sys, urllib.parse, urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
form = urllib.request.urlopen(url + "server/index.html", timeout=20).read()
|
||||
m = re.search(rb'name="sid" value="([0-9a-f]+)"', form)
|
||||
if not m:
|
||||
sys.exit("FAIL: no session id in server/index.html")
|
||||
fields = [("sid", m.group(1).decode()), ("singlefile", "on"),
|
||||
("singlefilemax", "5000")]
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v)) for k, v in fields)
|
||||
req = urllib.request.Request(url + "server/step4.html",
|
||||
data=body.encode("latin-1"), method="POST")
|
||||
page = urllib.request.urlopen(req, timeout=20).read().decode("latin-1")
|
||||
|
||||
|
||||
def textarea(name):
|
||||
m = re.search(r'<textarea name="%s".*?>(.*?)</textarea>' % name, page, re.S)
|
||||
if m is None:
|
||||
sys.exit("FAIL: no %s textarea in the rendered step4.html" % name)
|
||||
return m.group(1)
|
||||
|
||||
|
||||
cmd = textarea("command")
|
||||
for want in ("--single-file", "--single-file-max-size=5000"):
|
||||
if want not in cmd:
|
||||
sys.exit("FAIL: %s missing from the command line\n%s" % (want, cmd))
|
||||
ini = textarea("winprofile").replace("\r\n", "\n")
|
||||
for want in ("\nSingleFile=1\n", "\nSingleFileMaxSize=5000\n"):
|
||||
if want not in ini:
|
||||
sys.exit("FAIL: %r missing from winprofile.ini\n%s" % (want, ini))
|
||||
PY
|
||||
echo OK
|
||||
@@ -191,6 +191,7 @@ TESTS = \
|
||||
90_webhttrack-checkbox-clear.test \
|
||||
91_webhttrack-directory.test \
|
||||
92_local-proxytrack-ndx-fields.test \
|
||||
93_local-changes.test
|
||||
93_local-changes.test \
|
||||
94_local-single-file.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -47,11 +47,14 @@ echo "stub browser invoked with: \$1" >&2
|
||||
# Also fetch an option page and require a rendered title='' tooltip: proves the
|
||||
# option template expands and the \${html:} filter escapes into the attribute.
|
||||
# option9 additionally proves the WARC and change-report controls render with
|
||||
# their expanded labels.
|
||||
# their expanded labels, and option2 the --single-file pair: on field names plus
|
||||
# the absence of an unexpanded key, since the default locale here is French.
|
||||
opturl="\${1%/}/server/option2.html"
|
||||
warcurl="\${1%/}/server/option9.html"
|
||||
if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html &&
|
||||
opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='" &&
|
||||
printf '%s' "\$opt" | grep -qaF 'name="singlefile"' && printf '%s' "\$opt" | grep -qaF 'name="singlefilemax"' &&
|
||||
! printf '%s' "\$opt" | grep -qaF '\${LANG_SINGLEFILE}' &&
|
||||
warc="\$(curl -fsSL --max-time 20 "\$warcurl")" && printf '%s' "\$warc" | grep -qaF 'name="warcfile"' && printf '%s' "\$warc" | grep -qaF WARC &&
|
||||
printf '%s' "\$warc" | grep -qaF 'name="changes"' && printf '%s' "\$warc" | grep -qaF hts-changes.json; then
|
||||
echo PASS >"$marker"
|
||||
|
||||
Reference in New Issue
Block a user