mirror of
https://github.com/xroche/httrack.git
synced 2026-07-27 19:12:54 +03:00
Compare commits
9 Commits
master
...
fix/inline
| Author | SHA1 | Date | |
|---|---|---|---|
|
0b0a409717 | ||
|
|
076fa7a75c | ||
|
|
3886e7855c | ||
|
|
efdb73084b | ||
|
|
384a3b0ad9 | ||
|
|
f5abb4f706 | ||
|
|
4da6d2dd19 | ||
|
|
8ab185df01 | ||
|
|
fc2cef5e87 |
@@ -5601,12 +5601,20 @@ static int sf_nesting(const char *hay, const char *mime) {
|
||||
static const char sf_png[] = "\x89PNG\r\n\x1a\n\x00\x01\x02\xff";
|
||||
#define SF_PNG_LEN 12
|
||||
|
||||
static const char sf_svg[] = "<svg><g id=\"icon-a\"/></svg>";
|
||||
|
||||
static const char sf_page[] =
|
||||
"<html><head>\n"
|
||||
"<link rel=\"stylesheet\" href=\"css/main.css\">\n"
|
||||
"<link rel=\"canonical\" href=\"other.html\">\n"
|
||||
"<title>t</title>\n"
|
||||
"<style>body { background: url(\"img/a%20b.png\"); }</style>\n"
|
||||
"<style>body { background: url(\"img/a%20b.png\"); }\n"
|
||||
/* Everything the escape set has to neutralise before a fragment reaches an
|
||||
unquoted url() token: whitespace, '<'/'>', then a quote, a paren, a
|
||||
backslash and a high byte together. */
|
||||
"b { background: url('img/sprite.svg#w x'); }\n"
|
||||
"i { background: url('img/sprite.svg#lt<s>gt'); }\n"
|
||||
"u { background: url(\"img/sprite.svg#z'(\\\xC3\xA9\"); }</style>\n"
|
||||
"</head><body>\n"
|
||||
"<img src=\"img/a%20b.png\" srcset=\"img/a%20b.png 1x, img/big.png 2x\">\n"
|
||||
"<link rel=\"icon\" href=\"icon.png\">\n"
|
||||
@@ -5638,6 +5646,19 @@ static const char sf_page[] =
|
||||
"<script src=\"js/app.js\"></script>\n"
|
||||
"<script>var s = \"</scripting>\"; var t = \"<img src='img/a%20b.png'>\";"
|
||||
"</script>\n"
|
||||
/* A fragment selects inside the asset, so it has to survive onto the
|
||||
data: URI; the query named the remote resource and must not. */
|
||||
"<img src=\"img/sprite.svg#icon-a\">\n"
|
||||
"<img srcset=\"img/sprite.svg#icon-b 2x\">\n"
|
||||
"<svg><image xlink:href=\"img/sprite.svg#icon-c\"/></svg>\n"
|
||||
"<div style=\"background:url(img/sprite.svg#icon-d)\"></div>\n"
|
||||
"<div style=\"background:url('img/sprite.svg#i)e')\"></div>\n"
|
||||
"<img src=\"img/sprite.svg?v=1#icon-f\">\n"
|
||||
/* Already carries the document's own escapes: re-encoding either would
|
||||
make the browser look for a different id. */
|
||||
"<img src=\"img/sprite.svg#g&h%2Di\">\n"
|
||||
/* A '"' left raw here would end the attribute the rewriter re-quotes. */
|
||||
"<img src='img/sprite.svg#q\"z'>\n"
|
||||
"<img src=\"missing.png\" >\n"
|
||||
"<!--><img src=\"img/a%20b.png\">\n"
|
||||
"<div style=\"background:url(img/a%20b.png)\"></div>\n"
|
||||
@@ -5656,6 +5677,10 @@ static void sf_fixture(const char *root) {
|
||||
"@font-face { font-family: f; src: url(../font/f.woff2); }\n"
|
||||
"body { background: url(../img/a b.png); }\n"
|
||||
"div { background: url(../img/big.png); }\n"
|
||||
"div.s { background: url(../img/big-sprite.svg#icon-g); }\n"
|
||||
/* A name whose escapes the rebase has to put back, unlike a fragment's;
|
||||
the '#' has to come back encoded or it reads as one. */
|
||||
"div.h { background: url(../img/b&c%25d%23e.png); }\n"
|
||||
"/* url(../img/never.png) */\n";
|
||||
static const char nested[] = "div { background: url(../../img/a b.png); }\n";
|
||||
static const char two[] = "p { background: url(../../img/a b.png); }\n";
|
||||
@@ -5675,6 +5700,9 @@ static void sf_fixture(const char *root) {
|
||||
sf_put(root, "js/app.js", js, sizeof(js) - 1);
|
||||
sf_put(root, "img/a b.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/big.png", big, sizeof(big));
|
||||
sf_put(root, "img/sprite.svg", sf_svg, sizeof(sf_svg) - 1);
|
||||
sf_put(root, "img/big-sprite.svg", big, sizeof(big));
|
||||
sf_put(root, "img/b&c%d#e.png", big, sizeof(big));
|
||||
sf_put(root, "img/in.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/po.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/sv.png", sf_png, SF_PNG_LEN);
|
||||
@@ -5811,6 +5839,29 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
|
||||
NULL,
|
||||
"quote inside a rewritten style attribute not escaped");
|
||||
|
||||
/* Fragments: kept on the replacement, escaped where they would close the
|
||||
url() token, and never joined by the query the mirrored name dropped. */
|
||||
sf_check(strstr(out, "img/sprite.svg") == NULL,
|
||||
"a fragment-bearing reference was left a link");
|
||||
sf_check(sf_count(out, "#icon-a\"") == 1, "img src fragment dropped");
|
||||
sf_check(sf_count(out, "#icon-b 2x\"") == 1, "srcset fragment dropped");
|
||||
sf_check(sf_count(out, "#icon-c\"") == 1, "xlink:href fragment dropped");
|
||||
sf_check(sf_count(out, "#icon-d)") == 1, "style url() fragment dropped");
|
||||
sf_check(sf_count(out, "#i%29e)") == 1,
|
||||
"a fragment closing the url() token was not escaped");
|
||||
sf_check(sf_count(out, "#icon-f\"") == 1, "fragment after a query dropped");
|
||||
sf_check(strstr(out, "?v=1") == NULL, "query carried onto the data: URI");
|
||||
sf_check(sf_count(out, "#g&h%2Di\"") == 1,
|
||||
"an escape the document already carried was encoded again");
|
||||
sf_check(sf_count(out, "#q%22z\"") == 1,
|
||||
"a quote in a fragment was not escaped");
|
||||
sf_check(sf_count(out, "#w%20x)") == 1,
|
||||
"whitespace in a fragment was not escaped");
|
||||
sf_check(sf_count(out, "#lt%3Cs%3Egt)") == 1,
|
||||
"'<'/'>' in a fragment were not escaped");
|
||||
sf_check(sf_count(out, "#z%27%28%5C%C3%A9)") == 1,
|
||||
"a quote, paren, backslash or high byte was left raw");
|
||||
|
||||
sf_check(strstr(out, "img/big.png 2x") != NULL, "over-cap asset inlined");
|
||||
sf_check(strstr(out, " 1x") != NULL, "srcset descriptor lost");
|
||||
sf_check(sf_count(out, "img/a%20b.png") ==
|
||||
@@ -5837,6 +5888,10 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
|
||||
root, so it has to come back out as img/big.png or it dangles. */
|
||||
sf_check(strstr(css, "url(img/big.png)") != NULL,
|
||||
"over-cap url() not rebased onto the page's directory");
|
||||
sf_check(strstr(css, "url(img/big-sprite.svg#icon-g)") != NULL,
|
||||
"a rebased url() lost its fragment");
|
||||
sf_check(strstr(css, "url(img/b%26c%25d%23e.png)") != NULL,
|
||||
"a rebased name came back unescaped");
|
||||
nested = sf_decode(css, "text/css", NULL);
|
||||
sf_check(nested != NULL &&
|
||||
strstr(nested, "url(data:image/png;base64,") != NULL,
|
||||
|
||||
@@ -92,6 +92,30 @@ static int sf_is_space(int c) {
|
||||
|
||||
static int sf_is_sep(int c) { return c == '/' || c == '\\'; }
|
||||
|
||||
/* Drop the whitespace surrounding a reference. */
|
||||
static void sf_trim(const char **ref, size_t *reflen) {
|
||||
while (*reflen > 0 && sf_is_space((unsigned char) **ref)) {
|
||||
(*ref)++;
|
||||
(*reflen)--;
|
||||
}
|
||||
while (*reflen > 0 && sf_is_space((unsigned char) (*ref)[*reflen - 1]))
|
||||
(*reflen)--;
|
||||
}
|
||||
|
||||
/* Text after a reference's first '#', or NULL when it carries no fragment. */
|
||||
static const char *sf_fragment(const char *ref, size_t reflen, size_t *len) {
|
||||
size_t i;
|
||||
|
||||
sf_trim(&ref, &reflen);
|
||||
for (i = 0; i < reflen; i++) {
|
||||
if (ref[i] == '#') {
|
||||
*len = reflen - i - 1;
|
||||
return ref + i + 1;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Case-insensitive equality between the span [p,p+n) and a lowercase literal.
|
||||
*/
|
||||
static hts_boolean sf_span_eq(const char *p, size_t n, const char *lit) {
|
||||
@@ -202,12 +226,7 @@ static hts_boolean sf_resolve(const sf_ctx *ctx, const char *base_dir,
|
||||
size_t i, n;
|
||||
int sp = 0, part;
|
||||
|
||||
while (reflen > 0 && sf_is_space((unsigned char) *ref)) {
|
||||
ref++;
|
||||
reflen--;
|
||||
}
|
||||
while (reflen > 0 && sf_is_space((unsigned char) ref[reflen - 1]))
|
||||
reflen--;
|
||||
sf_trim(&ref, &reflen);
|
||||
/* A mirrored name carries no query or fragment. */
|
||||
for (n = 0; n < reflen; n++) {
|
||||
if (ref[n] == '#' || ref[n] == '?')
|
||||
@@ -347,17 +366,20 @@ static hts_boolean sf_append_base64(String *out, char *data, size_t len) {
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Append a mirror-relative path, percent-escaping everything an unquoted CSS
|
||||
url() or an HTML attribute could choke on. */
|
||||
static void sf_append_escaped_path(String *out, const char *p) {
|
||||
/* Append [p,p+len), escaping what an unquoted CSS url(), an HTML attribute or
|
||||
an appended fragment could choke on. preencoded text already carries the
|
||||
document's own '%' and '&' escapes, and encoding those again changes it. */
|
||||
static void sf_append_escaped(String *out, const char *p, size_t len,
|
||||
hts_boolean preencoded) {
|
||||
static const char hex[] = "0123456789ABCDEF";
|
||||
size_t i;
|
||||
|
||||
for (i = 0; p[i] != '\0'; i++) {
|
||||
for (i = 0; i < len; i++) {
|
||||
const unsigned char c = (unsigned char) p[i];
|
||||
|
||||
if (c <= 32 || c >= 127 || c == '"' || c == '\'' || c == '(' || c == ')' ||
|
||||
c == '\\' || c == '<' || c == '>' || c == '&' || c == '%') {
|
||||
c == '\\' || c == '<' || c == '>' || c == '#' ||
|
||||
(!preencoded && (c == '%' || c == '&'))) {
|
||||
StringAddchar(*out, '%');
|
||||
StringAddchar(*out, hex[c >> 4]);
|
||||
StringAddchar(*out, hex[c & 15]);
|
||||
@@ -367,6 +389,13 @@ static void sf_append_escaped_path(String *out, const char *p) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Re-attach a fragment: it selects inside the asset (an SVG sprite id) rather
|
||||
than naming it, so the replacement needs it too. */
|
||||
static void sf_append_fragment(String *out, const char *frag, size_t len) {
|
||||
StringAddchar(*out, '#');
|
||||
sf_append_escaped(out, frag, len, HTS_TRUE);
|
||||
}
|
||||
|
||||
static void sf_warn_oversize(sf_ctx *ctx, const char *path, LLint size,
|
||||
LLint cap) {
|
||||
if (*ctx->warn_budget <= 0)
|
||||
@@ -399,7 +428,8 @@ static hts_boolean sf_inline(sf_ctx *ctx, const char *base_dir, const char *ref,
|
||||
String path = STRING_EMPTY;
|
||||
char mime[HTS_MIMETYPE_SIZE];
|
||||
char *body;
|
||||
size_t body_len = 0;
|
||||
size_t body_len = 0, frag_len = 0;
|
||||
const char *const frag = sf_fragment(ref, reflen, &frag_len);
|
||||
LLint size, cap;
|
||||
int cls;
|
||||
hts_boolean done = HTS_FALSE;
|
||||
@@ -457,6 +487,8 @@ static hts_boolean sf_inline(sf_ctx *ctx, const char *base_dir, const char *ref,
|
||||
StringCat(*out, mime);
|
||||
StringCat(*out, ";base64,");
|
||||
StringMemcat(*out, StringBuff(payload), StringLength(payload));
|
||||
if (frag != NULL)
|
||||
sf_append_fragment(out, frag, frag_len);
|
||||
}
|
||||
StringFree(payload);
|
||||
}
|
||||
@@ -475,7 +507,9 @@ fallback:
|
||||
StringClear(rel);
|
||||
sf_relative_from(rebase_dir, StringBuff(path), &rel);
|
||||
if (StringLength(rel) > 0) {
|
||||
sf_append_escaped_path(out, StringBuff(rel));
|
||||
sf_append_escaped(out, StringBuff(rel), StringLength(rel), HTS_FALSE);
|
||||
if (frag != NULL)
|
||||
sf_append_fragment(out, frag, frag_len);
|
||||
done = HTS_TRUE;
|
||||
}
|
||||
StringFree(rel);
|
||||
|
||||
108
tests/112_local-single-file-fragment.test
Normal file
108
tests/112_local-single-file-fragment.test
Normal file
@@ -0,0 +1,108 @@
|
||||
#!/bin/bash
|
||||
# Issue #766: --single-file substituted the whole reference, dropping the '#'
|
||||
# fragment an SVG sprite selector needs to pick one icon out of the sheet.
|
||||
set -e
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_766.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- the site ---------------------------------------------------------------
|
||||
doc="${tmpdir}/doc"
|
||||
mkdir -p "$doc"
|
||||
cat >"${doc}/index.html" <<'EOF'
|
||||
<html><head><link rel="stylesheet" href="style.css"></head><body>
|
||||
<img src="sprite.svg#icon-foo" alt="f">
|
||||
<img srcset="sprite.svg#icon-set 2x" alt="s">
|
||||
</body></html>
|
||||
EOF
|
||||
cat >"${doc}/style.css" <<'EOF'
|
||||
body { background: url(sprite.svg#icon-css); }
|
||||
div { background: url(big.svg#icon-big); }
|
||||
EOF
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><g id="icon-foo"/></svg>\n' \
|
||||
>"${doc}/sprite.svg"
|
||||
{
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><!--'
|
||||
head -c 20000 /dev/zero | tr '\0' 'x'
|
||||
printf '%s\n' '--><g id="icon-big"/></svg>'
|
||||
} >"${doc}/big.svg"
|
||||
|
||||
# --- server -----------------------------------------------------------------
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
"$python" "$server" --root "$(nativepath "$doc")" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || fail "server exited: $(cat "$serverlog")"
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || fail "could not discover server port"
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || fail "could not find httrack"
|
||||
|
||||
# --- crawl ------------------------------------------------------------------
|
||||
# The cap sits between sprite.svg and big.svg, so one inlines and one is left
|
||||
# for the rebase path.
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
httrack -O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 \
|
||||
--retries=0 --single-file --single-file-max-size 4096 \
|
||||
"${base}/index.html" >"${tmpdir}/log" 2>&1
|
||||
|
||||
root="${out}/127.0.0.1_${port}"
|
||||
page="${root}/index.html"
|
||||
test -f "$page" || fail "no mirrored page at $page"
|
||||
test -f "${root}/big.svg" || fail "the over-cap asset never reached the mirror"
|
||||
|
||||
b64='[A-Za-z0-9+/=]+'
|
||||
|
||||
printf '[fragment survives onto the data: URI] ..\t'
|
||||
# Matching the data: prefix as well is what makes this more than "#icon-foo is
|
||||
# somewhere in the page": an un-inlined reference would carry it too.
|
||||
grep -Eq "src=\"data:image/svg\\+xml;base64,${b64}#icon-foo\"" "$page" ||
|
||||
fail "img src lost its fragment: $(grep -o 'src="[^"]*"' "$page" | head -n2)"
|
||||
grep -Eq "srcset=\"data:image/svg\\+xml;base64,${b64}#icon-set 2x\"" "$page" ||
|
||||
fail "srcset lost its fragment or its descriptor"
|
||||
echo OK
|
||||
|
||||
printf '[fragment survives inside an inlined stylesheet] ..\t'
|
||||
"$python" - "$page" "${tmpdir}/got.css" <<'PY' || fail "no inlined stylesheet"
|
||||
import base64, re, sys
|
||||
html = open(sys.argv[1], "rb").read().decode("utf-8", "replace")
|
||||
m = re.search(r'data:text/css;base64,([A-Za-z0-9+/=]*)', html)
|
||||
if m is None:
|
||||
sys.exit(1)
|
||||
open(sys.argv[2], "wb").write(base64.b64decode(m.group(1)))
|
||||
PY
|
||||
grep -Eq "url\\(data:image/svg\\+xml;base64,${b64}#icon-css\\)" "${tmpdir}/got.css" ||
|
||||
fail "url() lost its fragment: $(cat "${tmpdir}/got.css")"
|
||||
# Over the cap, so it stays a link, rebased onto the page's directory: it still
|
||||
# names the sprite sheet, so it still needs the selector.
|
||||
grep -q 'url(big.svg#icon-big)' "${tmpdir}/got.css" ||
|
||||
fail "the rebased over-cap url() lost its fragment"
|
||||
echo OK
|
||||
@@ -230,6 +230,7 @@ TESTS = \
|
||||
105_suite-timeout.test \
|
||||
110_local-ftp-parallel.test \
|
||||
111_local-ftp-update-rest.test \
|
||||
112_local-single-file-fragment.test \
|
||||
114_local-update-304-leak.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
Reference in New Issue
Block a user