mirror of
https://github.com/xroche/httrack.git
synced 2026-06-20 17:18:14 +03:00
Compare commits
16 Commits
fix/xmlns-
...
chore/lint
| Author | SHA1 | Date | |
|---|---|---|---|
|
fe7041ddbf | ||
|
|
f5543df1af | ||
|
|
fee30aa95d | ||
|
|
f9f4700ee1 | ||
|
|
f030fa21e3 | ||
|
|
bdd1c1bc2c | ||
|
|
56665a268f | ||
|
|
2e948b9acd | ||
|
|
cae11499f1 | ||
|
|
02c7f4ebf6 | ||
|
|
9070b44a70 | ||
|
|
799c045061 | ||
|
|
fb1ee3bf2e | ||
|
|
6a08ca7d39 | ||
|
|
a8b491e509 | ||
|
|
a8e4bb3b81 |
20
.github/workflows/ci.yml
vendored
20
.github/workflows/ci.yml
vendored
@@ -320,6 +320,21 @@ jobs:
|
||||
lint:
|
||||
name: lint (shellcheck, shfmt)
|
||||
runs-on: ubuntu-24.04
|
||||
# Every tracked shell script; the globs expand at run time. Kept here so the
|
||||
# shellcheck and shfmt steps below cannot drift apart.
|
||||
env:
|
||||
SHELL_SCRIPTS: >-
|
||||
.githooks/pre-commit
|
||||
bootstrap
|
||||
build.sh
|
||||
html/div/search.sh
|
||||
man/makeman.sh
|
||||
src/htsbasiccharsets.sh
|
||||
src/htsentities.sh
|
||||
src/webhttrack
|
||||
tests/*.sh
|
||||
tests/*.test
|
||||
tools/mkdeb.sh
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
@@ -332,12 +347,11 @@ jobs:
|
||||
sudo apt-get install -y --no-install-recommends shellcheck shfmt
|
||||
shfmt --version
|
||||
|
||||
# Lint the scripts we maintain; the legacy scripts are a separate cleanup.
|
||||
- name: shellcheck
|
||||
run: shellcheck man/makeman.sh tools/mkdeb.sh .githooks/pre-commit tests/*.test tests/check-network.sh
|
||||
run: shellcheck $SHELL_SCRIPTS
|
||||
|
||||
- name: shfmt
|
||||
run: shfmt -d -i 4 man/makeman.sh tools/mkdeb.sh .githooks/pre-commit
|
||||
run: shfmt -d -i 4 $SHELL_SCRIPTS
|
||||
|
||||
# Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
|
||||
# (it was shaped by an old Visual Studio formatter) and does not round-trip,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
|
||||
#!/bin/sh
|
||||
|
||||
# Simple indexing test using HTTrack
|
||||
# A "real" script/program would use advanced search, and
|
||||
# A "real" script/program would use advanced search, and
|
||||
# use dichotomy to find the word in the index.txt file
|
||||
# This script is really basic and NOT optimized, and
|
||||
# should not be used for professional purpose :)
|
||||
@@ -11,50 +10,49 @@ TESTSITE="http://localhost/"
|
||||
|
||||
# Create an index if necessary
|
||||
if ! test -f "index.txt"; then
|
||||
echo "Building the index .."
|
||||
rm -rf test
|
||||
httrack --display "$TESTSITE" -%I -O test
|
||||
mv test/index.txt ./
|
||||
echo "Building the index .."
|
||||
rm -rf test
|
||||
httrack --display "$TESTSITE" -%I -O test
|
||||
mv test/index.txt ./
|
||||
fi
|
||||
|
||||
# Convert crlf to lf
|
||||
if test "`head index.txt -n 1 | tr '\r' '#' | grep -c '#'`" = "1"; then
|
||||
echo "Converting index to Unix LF style (not CR/LF) .."
|
||||
mv -f index.txt index.txt.old
|
||||
cat index.txt.old|tr -d '\r' > index.txt
|
||||
if test "$(head index.txt -n 1 | tr '\r' '#' | grep -c '#')" = "1"; then
|
||||
echo "Converting index to Unix LF style (not CR/LF) .."
|
||||
mv -f index.txt index.txt.old
|
||||
tr -d '\r' <index.txt.old >index.txt
|
||||
fi
|
||||
|
||||
keyword=-
|
||||
while test -n "$keyword"; do
|
||||
printf "Enter a keyword: "
|
||||
read keyword
|
||||
printf "Enter a keyword: "
|
||||
read -r keyword
|
||||
|
||||
if test -n "$keyword"; then
|
||||
FOUNDK="`grep -niE \"^$keyword\" index.txt`"
|
||||
if test -n "$keyword"; then
|
||||
FOUNDK="$(grep -niE "^$keyword" index.txt)"
|
||||
|
||||
if test -n "$FOUNDK"; then
|
||||
if ! test `echo "$FOUNDK"|wc -l` = "1"; then
|
||||
# Multiple matches
|
||||
printf "Found multiple keywords: "
|
||||
echo "$FOUNDK"|cut -f2 -d':'|tr '\n' ' '
|
||||
echo ""
|
||||
echo "Use keyword$ to find only one"
|
||||
else
|
||||
# One match
|
||||
N=`echo "$FOUNDK"|cut -f1 -d':'`
|
||||
PM=`tail +$N index.txt|grep -nE "\("|head -n 1`
|
||||
if ! echo "$PM"|grep "ignored">/dev/null; then
|
||||
M=`echo $PM|cut -f1 -d':'`
|
||||
echo "Found in:"
|
||||
cat index.txt | tail "+$N" | head -n "$M" | grep -E "[0-9]* " | cut -f2 -d' '
|
||||
else
|
||||
echo "keyword ignored (too many hits)"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "not found"
|
||||
fi
|
||||
if test -n "$FOUNDK"; then
|
||||
if ! test "$(echo "$FOUNDK" | wc -l)" = "1"; then
|
||||
# Multiple matches
|
||||
printf "Found multiple keywords: "
|
||||
echo "$FOUNDK" | cut -f2 -d':' | tr '\n' ' '
|
||||
echo ""
|
||||
echo "Use keyword$ to find only one"
|
||||
else
|
||||
# One match
|
||||
N=$(echo "$FOUNDK" | cut -f1 -d':')
|
||||
PM=$(tail "+$N" index.txt | grep -nE "\(" | head -n 1)
|
||||
if ! echo "$PM" | grep "ignored" >/dev/null; then
|
||||
M=$(echo "$PM" | cut -f1 -d':')
|
||||
echo "Found in:"
|
||||
tail "+$N" index.txt | head -n "$M" | grep -E "[0-9]* " | cut -f2 -d' '
|
||||
else
|
||||
echo "keyword ignored (too many hits)"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "not found"
|
||||
fi
|
||||
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
@@ -3,57 +3,59 @@
|
||||
|
||||
# Change this to download files
|
||||
if false; then
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
|
||||
rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
|
||||
rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
|
||||
fi
|
||||
|
||||
# Produce code
|
||||
printf "/** GENERATED FILE ($0), DO NOT EDIT **/\n\n"
|
||||
for i in *.TXT ; do
|
||||
echo "processing $i" >&2
|
||||
grep -vE "^(#|$)" $i | grep -E "^0x" | sed -e 's/[[:space:]]/ /g' | cut -f1,2 -d' ' | \
|
||||
(
|
||||
unset arr
|
||||
while read LINE ; do
|
||||
from=$[$(echo $LINE | cut -f1 -d' ')]
|
||||
if ! test -n "$from"; then
|
||||
echo "error with $i" >&2
|
||||
exit 1
|
||||
elif test $from -ge 256; then
|
||||
echo "out-of-range ($LINE) with $i" >&2
|
||||
exit 1
|
||||
fi
|
||||
to=$(echo $LINE | cut -f2 -d' ')
|
||||
arr[$from]=$to
|
||||
done
|
||||
name=$(echo $i | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
|
||||
printf "/* Table for $i */\nstatic const hts_UCS4 table_${name}[256] = {\n "
|
||||
i=0
|
||||
while test "$i" -lt 256; do
|
||||
if test "$i" -gt 0; then
|
||||
printf ", "
|
||||
if test $[${i}%8] -eq 0; then
|
||||
printf "\n "
|
||||
fi
|
||||
fi
|
||||
value=${arr[$i]:-0}
|
||||
printf "0x%04x" $value
|
||||
i=$[${i}+1]
|
||||
done
|
||||
printf " };\n\n"
|
||||
)
|
||||
echo "processed $i" >&2
|
||||
printf '/** GENERATED FILE (%s), DO NOT EDIT **/\n\n' "$0"
|
||||
for i in *.TXT; do
|
||||
echo "processing $i" >&2
|
||||
grep -vE "^(#|$)" "$i" | grep -E "^0x" | sed -e 's/[[:space:]]/ /g' | cut -f1,2 -d' ' |
|
||||
(
|
||||
unset arr
|
||||
while read -r LINE; do
|
||||
from=$(($(echo "$LINE" | cut -f1 -d' ')))
|
||||
if ! test -n "$from"; then
|
||||
echo "error with $i" >&2
|
||||
exit 1
|
||||
elif test $from -ge 256; then
|
||||
echo "out-of-range ($LINE) with $i" >&2
|
||||
exit 1
|
||||
fi
|
||||
to=$(echo "$LINE" | cut -f2 -d' ')
|
||||
arr[from]=$to
|
||||
done
|
||||
# shellcheck disable=SC2018,SC2019 # charset filenames are ASCII; keep C-locale A-Z/a-z
|
||||
name=$(echo "$i" | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
|
||||
printf '/* Table for %s */\nstatic const hts_UCS4 table_%s[256] = {\n ' "$i" "$name"
|
||||
idx=0
|
||||
while test "$idx" -lt 256; do
|
||||
if test "$idx" -gt 0; then
|
||||
printf ", "
|
||||
if test $((idx % 8)) -eq 0; then
|
||||
printf "\n "
|
||||
fi
|
||||
fi
|
||||
value=${arr[$idx]:-0}
|
||||
printf "0x%04x" "$value"
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
printf " };\n\n"
|
||||
)
|
||||
echo "processed $i" >&2
|
||||
done
|
||||
|
||||
# Indexes
|
||||
printf "static const struct {\n const char *name;\n const hts_UCS4 *table;\n} table_mappings[] = {\n"
|
||||
for i in *.TXT ; do
|
||||
name=$(echo $i | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
|
||||
printf " { \"$(echo $name | tr -d '_')\", table_${name} },\n"
|
||||
for i in *.TXT; do
|
||||
# shellcheck disable=SC2018,SC2019 # charset filenames are ASCII; keep C-locale A-Z/a-z
|
||||
name=$(echo "$i" | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
|
||||
printf ' { "%s", table_%s },\n' "$(echo "$name" | tr -d '_')" "$name"
|
||||
done
|
||||
printf " { NULL, NULL }\n};\n"
|
||||
|
||||
@@ -2787,6 +2787,47 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
return 0;
|
||||
}
|
||||
break;
|
||||
case 'l': /* lienrelatif: relative link from curr_fil to link */
|
||||
if (na + 2 >= argc) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option #l needs a link and a current-file path");
|
||||
printf(
|
||||
"Example: '-#l' 'host/dir/img.gif' 'host/dir/p.html'\n");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
char s[HTS_URLMAXSIZE * 2];
|
||||
|
||||
if (lienrelatif(s, sizeof(s), argv[na + 1], argv[na + 2]) ==
|
||||
0)
|
||||
printf("relative=%s\n", s);
|
||||
else
|
||||
printf("relative=<ERROR>\n");
|
||||
htsmain_free();
|
||||
return 0;
|
||||
}
|
||||
break;
|
||||
case 'i': /* ident_url_relatif: resolve a link -> adr/fil */
|
||||
if (na + 3 >= argc) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option #i needs a link, an origin address and file");
|
||||
printf("Example: '-#i' '../img.gif' 'www.foo.com' "
|
||||
"'/d/p.html'\n");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
lien_adrfil af;
|
||||
const int r = ident_url_relatif(argv[na + 1], argv[na + 2],
|
||||
argv[na + 3], &af);
|
||||
|
||||
if (r == 0)
|
||||
printf("adr=%s fil=%s\n", af.adr, af.fil);
|
||||
else
|
||||
printf("error=%d\n", r);
|
||||
htsmain_free();
|
||||
return 0;
|
||||
}
|
||||
break;
|
||||
case '2': // mimedefs
|
||||
if (na + 1 >= argc) {
|
||||
HTS_PANIC_PRINTF("Option #2 needs to be followed by an URL");
|
||||
|
||||
@@ -33,43 +33,43 @@ EOF
|
||||
else
|
||||
GET "${url}"
|
||||
fi
|
||||
) \
|
||||
| grep -E '^<!ENTITY [a-zA-Z0-9_]' \
|
||||
| sed \
|
||||
-e 's/<!ENTITY //' -e "s/[[:space:]][[:space:]]*/ /g" \
|
||||
-e 's/-->$//' \
|
||||
-e 's/\([^ ]*\) CDATA "&#\([^\"]*\);" -- \(.*\)/\1 \2 \3/'\
|
||||
| ( \
|
||||
read A
|
||||
while test -n "$A"; do
|
||||
ent="${A%% *}"
|
||||
code=$(echo "$A"|cut -f2 -d' ')
|
||||
# compute hash
|
||||
hash=0
|
||||
i=0
|
||||
a=1664525
|
||||
c=1013904223
|
||||
m="$[1 << 32]"
|
||||
while test "$i" -lt ${#ent}; do
|
||||
d="$(echo -n "${ent:${i}:1}"|hexdump -v -e '/1 "%d"')"
|
||||
hash="$[((${hash}*${a})%(${m})+${d}+${c})%(${m})]"
|
||||
i=$[${i}+1]
|
||||
done
|
||||
echo -e " /* $A */"
|
||||
echo -e " case ${hash}u:"
|
||||
echo -e " if (len == ${#ent} /* && strncmp(ent, \"${ent}\") == 0 */) {"
|
||||
echo -e " return ${code};"
|
||||
echo -e " }"
|
||||
echo -e " break;"
|
||||
) |
|
||||
grep -E '^<!ENTITY [a-zA-Z0-9_]' |
|
||||
sed \
|
||||
-e 's/<!ENTITY //' -e "s/[[:space:]][[:space:]]*/ /g" \
|
||||
-e 's/-->$//' \
|
||||
-e 's/\([^ ]*\) CDATA "&#\([^\"]*\);" -- \(.*\)/\1 \2 \3/' |
|
||||
(
|
||||
read -r A
|
||||
while test -n "$A"; do
|
||||
ent="${A%% *}"
|
||||
code=$(echo "$A" | cut -f2 -d' ')
|
||||
# compute hash
|
||||
hash=0
|
||||
i=0
|
||||
a=1664525
|
||||
c=1013904223
|
||||
m="$((1 << 32))"
|
||||
while test "$i" -lt ${#ent}; do
|
||||
d="$(echo -n "${ent:${i}:1}" | hexdump -v -e '/1 "%d"')"
|
||||
hash="$((((hash * a) % (m) + d + c) % (m)))"
|
||||
i=$((i + 1))
|
||||
done
|
||||
echo -e " /* $A */"
|
||||
echo -e " case ${hash}u:"
|
||||
echo -e " if (len == ${#ent} /* && strncmp(ent, \"${ent}\") == 0 */) {"
|
||||
echo -e " return ${code};"
|
||||
echo -e " }"
|
||||
echo -e " break;"
|
||||
|
||||
# next
|
||||
read A
|
||||
done
|
||||
)
|
||||
# next
|
||||
read -r A
|
||||
done
|
||||
)
|
||||
cat <<EOF
|
||||
}
|
||||
/* unknown */
|
||||
return -1;
|
||||
}
|
||||
EOF
|
||||
) > ${dest}
|
||||
) >${dest}
|
||||
|
||||
@@ -2605,6 +2605,8 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
|
||||
for(i = 0; adrfil->fil[i] != '\0'; i++)
|
||||
if (adrfil->fil[i] == '\\')
|
||||
adrfil->fil[i] = '/';
|
||||
// collapse ../ like the http branch above (path-traversal safety)
|
||||
fil_simplifie(adrfil->fil);
|
||||
}
|
||||
|
||||
// no hostname
|
||||
|
||||
@@ -296,6 +296,48 @@ static const char *html_inline_safe(const char *src, char *dst, size_t size) {
|
||||
return dst;
|
||||
}
|
||||
|
||||
/* Byte before html, or a space sentinel at the buffer start where html[-1]
|
||||
would underflow; space reads as the word boundary the guards want there. */
|
||||
static HTS_INLINE char html_prevc(const char *html, const char *start) {
|
||||
return html > start ? html[-1] : ' ';
|
||||
}
|
||||
|
||||
/* True if [s, s+len) is exactly an HTTP method token (XHR.open's first
|
||||
argument is a method, not a URL: #218). Case-insensitive. */
|
||||
static int is_http_method(const char *s, size_t len) {
|
||||
static const char *const methods[] = {"GET", "POST", "PUT",
|
||||
"DELETE", "HEAD", "OPTIONS",
|
||||
"PATCH", "TRACE", NULL};
|
||||
int i;
|
||||
|
||||
for (i = 0; methods[i] != NULL; i++) {
|
||||
if (strlen(methods[i]) == len && strfield(s, methods[i]) == (int) len)
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Percent-encode '(' and ')' in a link emitted into an unquoted url(...) (CSS
|
||||
or JS): a literal ')' closes the token early and the UA mis-parses the value
|
||||
(#163). The UA decodes %28/%29 back to the saved-on-disk name. */
|
||||
static void escape_url_parens(char *const s, const size_t size) {
|
||||
char BIGSTK buff[HTS_URLMAXSIZE * 2];
|
||||
size_t i, j;
|
||||
|
||||
for (i = 0, j = 0; s[i] != '\0' && j + 3 < size && j + 3 < sizeof(buff);
|
||||
i++) {
|
||||
if (s[i] == '(' || s[i] == ')') {
|
||||
buff[j++] = '%';
|
||||
buff[j++] = '2';
|
||||
buff[j++] = s[i] == '(' ? '8' : '9';
|
||||
} else {
|
||||
buff[j++] = s[i];
|
||||
}
|
||||
}
|
||||
buff[j] = '\0';
|
||||
strlcpybuff(s, buff, size);
|
||||
}
|
||||
|
||||
/* Main parser */
|
||||
int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
@@ -556,7 +598,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (opt->getmode & HTS_GETMODE_HTML) {
|
||||
p = strfield(html, "title");
|
||||
if (p) {
|
||||
if (*(html - 1) == '/')
|
||||
if (html_prevc(html, r->adr) == '/')
|
||||
p = 0; // /title
|
||||
} else {
|
||||
if (strfield(html, "/html"))
|
||||
@@ -1341,6 +1383,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
int can_avoid_quotes = 0;
|
||||
char quotes_replacement = '\0';
|
||||
int ensure_not_mime = 0;
|
||||
// .open(method,url): reject an HTTP-method first arg (#218)
|
||||
int ensure_not_method = 0;
|
||||
// @import: the quoted token is the URL; a trailing
|
||||
// media/supports/layer condition is not part of it
|
||||
int is_import = 0;
|
||||
|
||||
if (inscript_tag)
|
||||
expected_end = ";\"\'"; // voir a href="javascript:doc.location='foo'"
|
||||
@@ -1357,9 +1404,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (!nc)
|
||||
nc = strfield(html, ":location"); // javascript:location="doc"
|
||||
if (!nc) { // location="doc"
|
||||
if ((nc = strfield(html, "location"))
|
||||
&& !isspace(*(html - 1))
|
||||
)
|
||||
if ((nc = strfield(html, "location")) &&
|
||||
!isspace(html_prevc(html, r->adr)))
|
||||
nc = 0;
|
||||
}
|
||||
if (!nc)
|
||||
@@ -1369,6 +1415,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
expected = '('; // parenthèse
|
||||
expected_end = "),"; // fin: virgule ou parenthèse
|
||||
ensure_not_mime = 1; //* ensure the url is not a mime type */
|
||||
ensure_not_method = 1; // xhr.open: don't grab method
|
||||
}
|
||||
if (!nc)
|
||||
if ((nc = strfield(html, ".replace"))) { // window.replace("url")
|
||||
@@ -1380,7 +1427,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
expected = '('; // parenthèse
|
||||
expected_end = ")"; // fin: parenthèse
|
||||
}
|
||||
if (!nc && (nc = strfield(html, "url")) && (!isalnum(*(html - 1))) && *(html - 1) != '_') { // url(url)
|
||||
if (!nc && (nc = strfield(html, "url")) &&
|
||||
(!isalnum(html_prevc(html, r->adr))) &&
|
||||
html_prevc(html, r->adr) != '_') { // url(url)
|
||||
expected = '('; // parenthèse
|
||||
expected_end = ")"; // fin: parenthèse
|
||||
can_avoid_quotes = 1;
|
||||
@@ -1390,6 +1439,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if ((nc = strfield(html, "import"))) { // import "url"
|
||||
if (is_space(*(html + nc))) {
|
||||
expected = 0; // no char expected
|
||||
is_import = 1;
|
||||
} else
|
||||
nc = 0;
|
||||
}
|
||||
@@ -1407,6 +1457,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if ((*a == 34) || (*a == '\'') || (can_avoid_quotes)) {
|
||||
const char *b, *c;
|
||||
int ndelim = 1;
|
||||
int valid_url = 0;
|
||||
|
||||
if ((*a == 34) || (*a == '\''))
|
||||
a++;
|
||||
@@ -1421,12 +1472,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
b++;
|
||||
}
|
||||
c = b--;
|
||||
c += ndelim;
|
||||
while(*c == ' ')
|
||||
c++;
|
||||
if ((strchr(expected_end, *c)) || (*c == '\n')
|
||||
|| (*c == '\r')) {
|
||||
c -= (ndelim + 1);
|
||||
// no closing delimiter here (truncated input):
|
||||
// Don't scan past the buffer NUL or capture it.
|
||||
if (*c != '\0') {
|
||||
c += ndelim;
|
||||
while (*c == ' ')
|
||||
c++;
|
||||
valid_url =
|
||||
(strchr(expected_end, *c)) || (*c == '\n') ||
|
||||
(*c == '\r') ||
|
||||
(is_import && *(b + 1 + ndelim) == ' ');
|
||||
}
|
||||
if (valid_url) {
|
||||
// URL end = last char (b), not the delimiter
|
||||
c = b;
|
||||
if ((int) (c - a + 1)) {
|
||||
if (ensure_not_mime) {
|
||||
int i = 0;
|
||||
@@ -1442,6 +1501,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
// XHR.open's "GET" etc. is a method, not a URL
|
||||
if (a != NULL && ensure_not_method &&
|
||||
is_http_method(a, (size_t) (c - a + 1))) {
|
||||
a = NULL;
|
||||
}
|
||||
// Check for bogus links (Vasiliy)
|
||||
if (a != NULL) {
|
||||
const size_t size = c - a + 1;
|
||||
@@ -1485,7 +1549,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2985,6 +3048,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
/* Never escape high-chars (we don't know the encoding!!) */
|
||||
inplace_escape_uri_utf(tempo, sizeof(tempo));
|
||||
|
||||
// unquoted url() (CSS/JS): keep parens escaped
|
||||
if (ending_p == ')')
|
||||
escape_url_parens(tempo, sizeof(tempo));
|
||||
|
||||
//if (!no_esc_utf)
|
||||
// escape_uri(tempo); // escape with %xx
|
||||
//else {
|
||||
|
||||
149
src/webhttrack
149
src/webhttrack
@@ -4,131 +4,140 @@
|
||||
# Initializes the htsserver GUI frontend and launch the default browser
|
||||
|
||||
BROWSEREXE=
|
||||
SRCHBROWSEREXE="x-www-browser www-browser iceape mozilla firefox-developer-edition firefox icecat iceweasel abrowser firebird galeon konqueror midori opera google-chrome chrome chromium chromium-browser netscape firefox-developer-edition"
|
||||
SRCHBROWSEREXE=(x-www-browser www-browser iceape mozilla firefox-developer-edition firefox icecat iceweasel abrowser firebird galeon konqueror midori opera google-chrome chrome chromium chromium-browser netscape firefox-developer-edition)
|
||||
# shellcheck disable=SC2153 # BROWSER is the standard freedesktop env var, not a typo
|
||||
if test -n "${BROWSER}"; then
|
||||
# sensible-browser will f up if BROWSER is not set
|
||||
SRCHBROWSEREXE="xdg-open sensible-browser ${SRCHBROWSEREXE}"
|
||||
# sensible-browser will f up if BROWSER is not set
|
||||
SRCHBROWSEREXE=(xdg-open sensible-browser "${SRCHBROWSEREXE[@]}")
|
||||
fi
|
||||
# Patch for Darwin/Mac by Ross Williams
|
||||
if test "`uname -s`" == "Darwin"; then
|
||||
# Darwin/Mac OS X uses a system 'open' command to find
|
||||
# the default browser. The -W flag causes it to wait for
|
||||
# the browser to exit
|
||||
BROWSEREXE="/usr/bin/open -W"
|
||||
if test "$(uname -s)" == "Darwin"; then
|
||||
# Darwin/Mac OS X uses a system 'open' command to find
|
||||
# the default browser. The -W flag causes it to wait for
|
||||
# the browser to exit
|
||||
BROWSEREXE="/usr/bin/open -W"
|
||||
fi
|
||||
BINWD=`dirname "$0"`
|
||||
SRCHPATH="$BINWD /usr/local/bin /usr/share/bin /usr/bin /usr/lib/httrack /usr/local/lib/httrack /usr/local/share/httrack /opt/local/bin /sw/bin ${HOME}/usr/bin ${HOME}/bin"
|
||||
SRCHPATH="$SRCHPATH "`echo $PATH | tr ":" " "`
|
||||
SRCHDISTPATH="$BINWD/../share $BINWD/.. /usr/share /usr/local /usr /local /usr/local/share ${HOME}/usr ${HOME}/usr/share /opt/local/share /sw ${HOME}/usr/local ${HOME}/usr/share"
|
||||
BINWD=$(dirname "$0")
|
||||
SRCHPATH=("$BINWD" /usr/local/bin /usr/share/bin /usr/bin /usr/lib/httrack /usr/local/lib/httrack /usr/local/share/httrack /opt/local/bin /sw/bin "${HOME}/usr/bin" "${HOME}/bin")
|
||||
IFS=':' read -ra pathdirs <<<"$PATH"
|
||||
for d in "${pathdirs[@]}"; do
|
||||
# drop empty PATH fields, matching the old echo|tr word-split
|
||||
test -n "$d" && SRCHPATH+=("$d")
|
||||
done
|
||||
SRCHDISTPATH=("$BINWD/../share" "$BINWD/.." /usr/share /usr/local /usr /local /usr/local/share "${HOME}/usr" "${HOME}/usr/share" /opt/local/share /sw "${HOME}/usr/local" "${HOME}/usr/share")
|
||||
|
||||
###
|
||||
# And now some famous cuisine
|
||||
|
||||
function log {
|
||||
echo "$0($$): $@" >&2
|
||||
return 0
|
||||
echo "$0($$): $*" >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
function launch_browser {
|
||||
log "Launching $1"
|
||||
browser=$1
|
||||
url=$2
|
||||
log "Spawning browser.."
|
||||
${browser} "${url}"
|
||||
# note: browser can hiddenly use the -remote feature of
|
||||
# mozilla and therefore return immediately
|
||||
log "Browser (or helper) exited"
|
||||
log "Launching $1"
|
||||
browser=$1
|
||||
url=$2
|
||||
log "Spawning browser.."
|
||||
${browser} "${url}"
|
||||
# note: browser can hiddenly use the -remote feature of
|
||||
# mozilla and therefore return immediately
|
||||
log "Browser (or helper) exited"
|
||||
}
|
||||
|
||||
# First ensure that we can launch the server
|
||||
BINPATH=
|
||||
for i in ${SRCHPATH}; do
|
||||
! test -n "${BINPATH}" && test -x ${i}/htsserver && BINPATH=${i}
|
||||
for i in "${SRCHPATH[@]}"; do
|
||||
! test -n "${BINPATH}" && test -x "${i}/htsserver" && BINPATH="${i}"
|
||||
done
|
||||
for i in ${SRCHDISTPATH}; do
|
||||
! test -n "${DISTPATH}" && test -f "${i}/httrack/lang.def" && DISTPATH="${i}/httrack"
|
||||
for i in "${SRCHDISTPATH[@]}"; do
|
||||
! test -n "${DISTPATH}" && test -f "${i}/httrack/lang.def" && DISTPATH="${i}/httrack"
|
||||
done
|
||||
test -n "${BINPATH}" || ! log "Could not find htsserver" || exit 1
|
||||
test -n "${DISTPATH}" || ! log "Could not find httrack directory" || exit 1
|
||||
test -f ${DISTPATH}/lang.def || ! log "Could not find ${DISTPATH}/lang.def" || exit 1
|
||||
test -f ${DISTPATH}/lang.indexes || ! log "Could not find ${DISTPATH}/lang.indexes" || exit 1
|
||||
test -d ${DISTPATH}/lang || ! log "Could not find ${DISTPATH}/lang" || exit 1
|
||||
test -d ${DISTPATH}/html || ! log "Could not find ${DISTPATH}/html" || exit 1
|
||||
test -f "${DISTPATH}/lang.def" || ! log "Could not find ${DISTPATH}/lang.def" || exit 1
|
||||
test -f "${DISTPATH}/lang.indexes" || ! log "Could not find ${DISTPATH}/lang.indexes" || exit 1
|
||||
test -d "${DISTPATH}/lang" || ! log "Could not find ${DISTPATH}/lang" || exit 1
|
||||
test -d "${DISTPATH}/html" || ! log "Could not find ${DISTPATH}/html" || exit 1
|
||||
|
||||
# Locale
|
||||
HTSLANG="${LC_MESSAGES}"
|
||||
! test -n "${HTSLANG}" && HTSLANG="${LC_ALL}"
|
||||
! test -n "${HTSLANG}" && HTSLANG="${LANG}"
|
||||
HTSLANG="`echo $LANG | cut -f1 -d'.' | cut -f1 -d'_'`"
|
||||
LANGN=`grep -E "^${HTSLANG}:" ${DISTPATH}/lang.indexes | cut -f2 -d':'`
|
||||
HTSLANG="$(echo "$LANG" | cut -f1 -d'.' | cut -f1 -d'_')"
|
||||
LANGN=$(grep -E "^${HTSLANG}:" "${DISTPATH}/lang.indexes" | cut -f2 -d':')
|
||||
! test -n "${LANGN}" && LANGN=1
|
||||
|
||||
# Find the browser
|
||||
# note: not all systems have sensible-browser or www-browser alternative
|
||||
# thefeore, we have to find a bit more if sensible-browser could not be found
|
||||
|
||||
for i in ${SRCHBROWSEREXE}; do
|
||||
for j in ${SRCHPATH}; do
|
||||
if test -x ${j}/${i}; then
|
||||
BROWSEREXE=${j}/${i}
|
||||
fi
|
||||
test -n "$BROWSEREXE" && break
|
||||
done
|
||||
test -n "$BROWSEREXE" && break
|
||||
for i in "${SRCHBROWSEREXE[@]}"; do
|
||||
for j in "${SRCHPATH[@]}"; do
|
||||
if test -x "${j}/${i}"; then
|
||||
BROWSEREXE="${j}/${i}"
|
||||
fi
|
||||
test -n "$BROWSEREXE" && break
|
||||
done
|
||||
test -n "$BROWSEREXE" && break
|
||||
done
|
||||
test -n "$BROWSEREXE" || ! log "Could not find any suitable browser" || exit 1
|
||||
|
||||
# "browse" command
|
||||
if test "$1" = "browse"; then
|
||||
if test -f "${HOME}/.httrack.ini"; then
|
||||
INDEXF=`cat ${HOME}/.httrack.ini | tr '\r' '\n' | grep -E "^path=" | cut -f2- -d'='`
|
||||
if test -n "${INDEXF}" -a -d "${INDEXF}" -a -f "${INDEXF}/index.html"; then
|
||||
INDEXF="${INDEXF}/index.html"
|
||||
else
|
||||
INDEXF=""
|
||||
fi
|
||||
fi
|
||||
if ! test -n "$INDEXF"; then
|
||||
INDEXF="${HOME}/websites/index.html"
|
||||
fi
|
||||
launch_browser "${BROWSEREXE}" "file://${INDEXF}"
|
||||
exit $?
|
||||
if test -f "${HOME}/.httrack.ini"; then
|
||||
INDEXF=$(tr '\r' '\n' <"${HOME}/.httrack.ini" | grep -E "^path=" | cut -f2- -d'=')
|
||||
if test -n "${INDEXF}" -a -d "${INDEXF}" -a -f "${INDEXF}/index.html"; then
|
||||
INDEXF="${INDEXF}/index.html"
|
||||
else
|
||||
INDEXF=""
|
||||
fi
|
||||
fi
|
||||
if ! test -n "$INDEXF"; then
|
||||
INDEXF="${HOME}/websites/index.html"
|
||||
fi
|
||||
launch_browser "${BROWSEREXE}" "file://${INDEXF}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Create a temporary filename
|
||||
TMPSRVFILE="$(mktemp ${TMPDIR:-/tmp}/.webhttrack.XXXXXXXX)" || ! log "Could not create the temporary file ${TMPSRVFILE}" || exit 1
|
||||
TMPSRVFILE="$(mktemp "${TMPDIR:-/tmp}/.webhttrack.XXXXXXXX")" || ! log "Could not create the temporary file ${TMPSRVFILE}" || exit 1
|
||||
# Launch htsserver binary and setup the server
|
||||
(${BINPATH}/htsserver "${DISTPATH}/" --ppid "$$" path "${HOME}/websites" lang "${LANGN}" $@; echo SRVURL=error) > ${TMPSRVFILE}&
|
||||
(
|
||||
"${BINPATH}/htsserver" "${DISTPATH}/" --ppid "$$" path "${HOME}/websites" lang "${LANGN}" "$@"
|
||||
echo SRVURL=error
|
||||
) >"${TMPSRVFILE}" &
|
||||
# Find the generated SRVURL
|
||||
SRVURL=
|
||||
MAXCOUNT=60
|
||||
while ! test -n "$SRVURL"; do
|
||||
MAXCOUNT=$[$MAXCOUNT - 1]
|
||||
test $MAXCOUNT -gt 0 || exit 1
|
||||
test $MAXCOUNT -lt 50 && echo "waiting for server to reply.."
|
||||
SRVURL=`grep -E URL= ${TMPSRVFILE} | cut -f2- -d=`
|
||||
test ! "$SRVURL" = "error" || ! log "Could not spawn htsserver" || exit 1
|
||||
test -n "$SRVURL" || sleep 1
|
||||
MAXCOUNT=$((MAXCOUNT - 1))
|
||||
test $MAXCOUNT -gt 0 || exit 1
|
||||
test $MAXCOUNT -lt 50 && echo "waiting for server to reply.."
|
||||
SRVURL=$(grep -E URL= "${TMPSRVFILE}" | cut -f2- -d=)
|
||||
test ! "$SRVURL" = "error" || ! log "Could not spawn htsserver" || exit 1
|
||||
test -n "$SRVURL" || sleep 1
|
||||
done
|
||||
|
||||
# Cleanup function
|
||||
# shellcheck disable=SC2120 # $1 is an optional "signal caught" marker; bare calls are intentional
|
||||
function cleanup {
|
||||
test -n "$1" && log "Nasty signal caught, cleaning up.."
|
||||
# Do not kill if browser exited (chrome bug issue) ; server will die itself
|
||||
test -n "$1" && test -f ${TMPSRVFILE} && SRVPID=`grep -E PID= ${TMPSRVFILE} | cut -f2- -d=`
|
||||
test -n "${SRVPID}" && kill -9 ${SRVPID}
|
||||
test -f ${TMPSRVFILE} && rm ${TMPSRVFILE}
|
||||
test -n "$1" && log "..Done"
|
||||
return 0
|
||||
test -n "$1" && log "Nasty signal caught, cleaning up.."
|
||||
# Do not kill if browser exited (chrome bug issue) ; server will die itself
|
||||
test -n "$1" && test -f "${TMPSRVFILE}" && SRVPID=$(grep -E PID= "${TMPSRVFILE}" | cut -f2- -d=)
|
||||
test -n "${SRVPID}" && kill -9 "${SRVPID}"
|
||||
test -f "${TMPSRVFILE}" && rm "${TMPSRVFILE}"
|
||||
test -n "$1" && log "..Done"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Cleanup in case of emergency
|
||||
trap "cleanup now; exit" 1 2 3 4 5 6 7 8 9 11 13 14 15 16 19 24 25
|
||||
trap "cleanup now; exit" HUP INT QUIT ILL TRAP ABRT BUS FPE SEGV PIPE ALRM TERM STKFLT XCPU XFSZ
|
||||
|
||||
# Got SRVURL, launch browser
|
||||
launch_browser "${BROWSEREXE}" "${SRVURL}"
|
||||
|
||||
# That's all, folks!
|
||||
trap "" 1 2 3 4 5 6 7 8 9 11 13 14 15 16 19 24 25
|
||||
trap "" HUP INT QUIT ILL TRAP ABRT BUS FPE SEGV PIPE ALRM TERM STKFLT XCPU XFSZ
|
||||
cleanup
|
||||
exit 0
|
||||
|
||||
@@ -6,11 +6,11 @@ set -euo pipefail
|
||||
# charset -> UTF-8 conversion (hts_convertStringToUTF8).
|
||||
# -#3 <charset> <string> prints the string re-decoded from <charset> as UTF-8.
|
||||
conv() {
|
||||
test "$(httrack -O /dev/null -#3 "$1" "$2")" == "$3" || exit 1
|
||||
test "$(httrack -O /dev/null -#3 "$1" "$2")" == "$3" || exit 1
|
||||
}
|
||||
# crash probe: malformed input must exit cleanly, not abort.
|
||||
runs() {
|
||||
httrack -O /dev/null -#3 "$1" "$2" >/dev/null 2>&1 || exit 1
|
||||
httrack -O /dev/null -#3 "$1" "$2" >/dev/null 2>&1 || exit 1
|
||||
}
|
||||
|
||||
# the source bytes below are UTF-8 (this file is UTF-8); "café" is 0x63 61 66 C3 A9.
|
||||
|
||||
@@ -6,11 +6,11 @@ set -euo pipefail
|
||||
# HTML entity unescaping (hts_unescapeEntitiesWithCharset).
|
||||
# -#6 <string> prints the string with entities decoded (UTF-8 output).
|
||||
ent() {
|
||||
test "$(httrack -O /dev/null -#6 "$1")" == "$2" || exit 1
|
||||
test "$(httrack -O /dev/null -#6 "$1")" == "$2" || exit 1
|
||||
}
|
||||
# crash probe: malformed input must exit cleanly, not abort.
|
||||
runs() {
|
||||
httrack -O /dev/null -#6 "$1" >/dev/null 2>&1 || exit 1
|
||||
httrack -O /dev/null -#6 "$1" >/dev/null 2>&1 || exit 1
|
||||
}
|
||||
|
||||
# named entities
|
||||
|
||||
@@ -7,10 +7,10 @@ set -euo pipefail
|
||||
# -#0 <filter> <string> prints "<string> does match <filter>" or "... does NOT match ...".
|
||||
|
||||
match() {
|
||||
test "$(httrack -O /dev/null -#0 "$1" "$2")" == "$2 does match $1" || exit 1
|
||||
test "$(httrack -O /dev/null -#0 "$1" "$2")" == "$2 does match $1" || exit 1
|
||||
}
|
||||
nomatch() {
|
||||
test "$(httrack -O /dev/null -#0 "$1" "$2")" == "$2 does NOT match $1" || exit 1
|
||||
test "$(httrack -O /dev/null -#0 "$1" "$2")" == "$2 does NOT match $1" || exit 1
|
||||
}
|
||||
|
||||
# bare star matches everything
|
||||
@@ -67,7 +67,7 @@ nomatch '*[\[]' 'a'
|
||||
# filter guide claims (GitHub #148); it parses as the class {'[','\'} followed
|
||||
# by a trailing literal ']'. These assertions document the current (buggy)
|
||||
# behavior so any future matcher fix is a deliberate, visible change.
|
||||
nomatch '*[\[\]]' '[' # not matched, despite the docs
|
||||
match '*[\[\]]' ']' # only via the empty class-match + trailing ']'
|
||||
match '*[\[\]]' '[]' # one of {'[','\'} then the trailing ']'
|
||||
nomatch '*[\[\]]' '[' # not matched, despite the docs
|
||||
match '*[\[\]]' ']' # only via the empty class-match + trailing ']'
|
||||
match '*[\[\]]' '[]' # one of {'[','\'} then the trailing ']'
|
||||
nomatch '*[\[\]]' '[]x'
|
||||
|
||||
@@ -7,10 +7,10 @@ set -euo pipefail
|
||||
# -#2 <path> prints "<path> is '<mime>'" then "and its local type is '.<ext>'".
|
||||
|
||||
mime() {
|
||||
test "$(httrack -O /dev/null -#2 "$1" | head -1)" == "$1 is '$2'" || exit 1
|
||||
test "$(httrack -O /dev/null -#2 "$1" | head -1)" == "$1 is '$2'" || exit 1
|
||||
}
|
||||
unknown() {
|
||||
test "$(httrack -O /dev/null -#2 "$1" | head -1)" == "$1 is of an unknown MIME type" || exit 1
|
||||
test "$(httrack -O /dev/null -#2 "$1" | head -1)" == "$1 is of an unknown MIME type" || exit 1
|
||||
}
|
||||
|
||||
mime '/a/b.html' 'text/html'
|
||||
|
||||
@@ -176,4 +176,151 @@ notfound "ns.gif" "$out3"
|
||||
notfound "og.gif" "$out3"
|
||||
notfound "rdfs.gif" "$out3"
|
||||
|
||||
# CSS @import (#94): every form's target is captured, crawling the .css directly.
|
||||
# The "cond"/"sup"/"spc" cases carry a trailing media/supports/layer condition (or
|
||||
# a space before ';'); they are the negative controls: without the parser fix the
|
||||
# URL is dropped, so a regression fails these found() checks.
|
||||
site4="$tmp/cssimport"
|
||||
mkdir -p "$site4"
|
||||
for f in nq dqu squ dqs sqs med cond sup lay spc; do printf 'body{}\n' >"$site4/$f.css"; done
|
||||
cat >"$site4/main.css" <<'EOF'
|
||||
@import url(nq.css);
|
||||
@import url("dqu.css");
|
||||
@import url('squ.css');
|
||||
@import "dqs.css";
|
||||
@import 'sqs.css';
|
||||
@import url(med.css) screen and (min-width: 400px);
|
||||
@import "cond.css" screen;
|
||||
@import "sup.css" supports(display: flex);
|
||||
@import url(lay.css) layer(base);
|
||||
@import "spc.css" ;
|
||||
EOF
|
||||
out4="$tmp/cssimport-out"
|
||||
crawl "$site4/main.css" "$out4"
|
||||
for f in nq dqu squ dqs sqs med cond sup lay spc; do found "$f.css" "$out4"; done
|
||||
|
||||
# Over-capture guard: the trailing condition is not part of the URL, so it must
|
||||
# survive the rewrite verbatim. A regression that grabs it would mangle these.
|
||||
m4=$(find "$out4" -type f -path '*/file/*' -name main.css -print -quit)
|
||||
test -n "$m4" || ! echo "FAIL: saved main.css not found" || exit 1
|
||||
for cond in '@import "cond.css" screen;' 'supports(display: flex)' 'layer(base)'; do
|
||||
grep -Fq "$cond" "$m4" ||
|
||||
! echo "FAIL #94: '$cond' altered on rewrite (condition captured as URL?)" || exit 1
|
||||
done
|
||||
|
||||
# Malformed input: an unterminated @import quote (truncated CSS) must not crash or
|
||||
# capture a bogus link; a valid sibling import is still captured. Guards a heap
|
||||
# overflow on the URL-end scan that aborts under ASan (CI sanitizer job).
|
||||
site5="$tmp/cssimport-trunc"
|
||||
mkdir -p "$site5"
|
||||
printf 'body{}\n' >"$site5/good.css"
|
||||
printf '@import "good.css";\n@import "trunc' >"$site5/main.css"
|
||||
out5="$tmp/cssimport-trunc-out"
|
||||
crawl "$site5/main.css" "$out5"
|
||||
found "good.css" "$out5"
|
||||
notfound "trunc" "$out5"
|
||||
|
||||
# Offset-0 underflow (#396): a token at the buffer start makes the detector's
|
||||
# word-boundary guard read *(html-1) one byte early (aborts under ASan). The
|
||||
# url() target is still captured; here it just must not underflow.
|
||||
site6="$tmp/parse-off0"
|
||||
mkdir -p "$site6"
|
||||
printf 'body{}\n' >"$site6/off0.css"
|
||||
printf 'url(off0.css)\n' >"$site6/main.css"
|
||||
out6="$tmp/parse-off0-out"
|
||||
crawl "$site6/main.css" "$out6"
|
||||
found "off0.css" "$out6"
|
||||
|
||||
# XMLHttpRequest.open(method, url) (#218): the first argument is an HTTP method,
|
||||
# not a URL. Without the fix "GET" is captured as a link and fetched (the offline
|
||||
# fixture saves a bare file named GET; a live server mangles it to GET.html).
|
||||
# window.open(url) detection must be unaffected.
|
||||
site7="$tmp/xhropen"
|
||||
mkdir -p "$site7"
|
||||
gif "$site7/winopen.gif"
|
||||
cat >"$site7/index.html" <<EOF
|
||||
<html><body><script>
|
||||
var x = new XMLHttpRequest();
|
||||
x.open("GET", "ajax_info.txt");
|
||||
var y = new XMLHttpRequest();
|
||||
y.open("Post", "submit.cgi");
|
||||
window.open("file://$site7/winopen.gif");
|
||||
</script></body></html>
|
||||
EOF
|
||||
out7="$tmp/xhropen-out"
|
||||
crawl "$site7/index.html" "$out7"
|
||||
# negative control: without the fix a file named exactly GET is downloaded
|
||||
notfound "GET" "$out7"
|
||||
# methods are matched case-insensitively (XHR spec normalizes them): a mixed-case
|
||||
# method is rejected too, so a file named Post must not appear either
|
||||
notfound "Post" "$out7"
|
||||
# regression guard: window.open(url) is still detected, so its absolute URL is
|
||||
# rewritten to a local link. The rewrite only happens if the parser saw it, so
|
||||
# these two assertions fail if .open detection broke (not a trivial --near save).
|
||||
saved7=$(savedhtml "$out7")
|
||||
test -n "$saved7" || ! echo "FAIL: saved xhr page not found" || exit 1
|
||||
grep -Fq 'window.open("winopen.gif")' "$saved7" ||
|
||||
! echo "FAIL #218: window.open(url) no longer detected/rewritten" || exit 1
|
||||
! grep -Fq 'window.open("file://' "$saved7" ||
|
||||
! echo "FAIL #218: window.open URL left absolute (not rewritten)" || exit 1
|
||||
|
||||
# Parens in an unquoted url(...) (#163): the source %28/%29 decode to literal
|
||||
# '(' ')' in the saved name, but a literal ')' in the rewritten url() closes the
|
||||
# token early, so they must stay encoded. Negative control: without the fix the
|
||||
# %281%29 greps fail (parens are RFC2396 "mark" chars the escaper leaves alone).
|
||||
site8="$tmp/cssparens"
|
||||
mkdir -p "$site8"
|
||||
for f in 'img (1).gif' 'a(b)c(1).gif' 'q (4).gif'; do gif "$site8/$f"; done
|
||||
cat >"$site8/style.css" <<'EOF'
|
||||
.a { background: url(img%20%281%29.gif); }
|
||||
.b { background: url(a%28b%29c%281%29.gif); }
|
||||
.c { background: url("q%20%284%29.gif"); }
|
||||
EOF
|
||||
out8="$tmp/cssparens-out"
|
||||
crawl "$site8/style.css" "$out8"
|
||||
found "img (1).gif" "$out8"
|
||||
found "a(b)c(1).gif" "$out8"
|
||||
found "q (4).gif" "$out8"
|
||||
css8=$(find "$out8" -type f -path '*/file/*' -name style.css -print -quit)
|
||||
test -n "$css8" || ! echo "FAIL: saved style.css not found" || exit 1
|
||||
grep -Fq 'url(img%20%281%29.gif)' "$css8" ||
|
||||
! echo "FAIL #163: parens in unquoted url() not percent-encoded on rewrite" || exit 1
|
||||
grep -Fq 'url(a%28b%29c%281%29.gif)' "$css8" ||
|
||||
! echo "FAIL #163: not every paren in a url() was percent-encoded" || exit 1
|
||||
grep -Fq 'url("q%20%284%29.gif")' "$css8" ||
|
||||
! echo "FAIL #163: quoted url() altered or parens left literal on rewrite" || exit 1
|
||||
|
||||
# The url() detector is not CSS-specific: <script> and inline style= get the
|
||||
# same encoding, but ordinary href/src (ending_p is the quote, not ')') keep
|
||||
# literal parens -- the attribute checks guard the gate against over-firing.
|
||||
site9="$tmp/urlparens"
|
||||
mkdir -p "$site9"
|
||||
for f in 'js (1).gif' 'inl (2).gif' 'asrc (3).gif' 'ahref (4).gif'; do gif "$site9/$f"; done
|
||||
cat >"$site9/index.html" <<EOF
|
||||
<html><body>
|
||||
<script>var bg = "url(js%20%281%29.gif)";</script>
|
||||
<div style="background-image:url(inl%20%282%29.gif)"></div>
|
||||
<img src="asrc%20%283%29.gif">
|
||||
<a href="ahref%20%284%29.gif">link</a>
|
||||
</body></html>
|
||||
EOF
|
||||
out9="$tmp/urlparens-out"
|
||||
crawl "$site9/index.html" "$out9"
|
||||
saved9=$(savedhtml "$out9")
|
||||
test -n "$saved9" || ! echo "FAIL: saved urlparens page not found" || exit 1
|
||||
# rewrite-only: the JS-string asset is not queued for download
|
||||
grep -Fq 'url(js%20%281%29.gif)' "$saved9" ||
|
||||
! echo "FAIL #163: parens in <script> url() not percent-encoded" || exit 1
|
||||
found "inl (2).gif" "$out9"
|
||||
grep -Fq 'url(inl%20%282%29.gif)' "$saved9" ||
|
||||
! echo "FAIL #163: parens in inline style url() not percent-encoded" || exit 1
|
||||
found "asrc (3).gif" "$out9"
|
||||
found "ahref (4).gif" "$out9"
|
||||
grep -Fq 'src="asrc%20(3).gif"' "$saved9" ||
|
||||
! echo "FAIL #163: parens in a plain src attribute were wrongly encoded" || exit 1
|
||||
grep -Fq 'href="ahref%20(4).gif"' "$saved9" ||
|
||||
! echo "FAIL #163: parens in a plain href attribute were wrongly encoded" || exit 1
|
||||
! grep -Eq '(src|href)="[^"]*%28' "$saved9" ||
|
||||
! echo "FAIL #163: gate over-fired onto a non-url() attribute link" || exit 1
|
||||
|
||||
exit 0
|
||||
|
||||
68
tests/01_engine-relative.test
Executable file
68
tests/01_engine-relative.test
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# lienrelatif (build relative path) + ident_url_relatif (resolve a link, collapse
|
||||
# ./ and ../). Regression net for #137/#162; expected values hand-computed.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# relative path from <curr>'s directory to <link>
|
||||
rel() {
|
||||
local got
|
||||
got=$(httrack -O /dev/null -#l "$1" "$2")
|
||||
test "$got" == "relative=$3" ||
|
||||
{
|
||||
echo "FAIL rel($1, $2): got '$got' want 'relative=$3'"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# resolve <link> against origin <adr>/<fil> -> adr=.. fil=..
|
||||
ident() {
|
||||
local got
|
||||
got=$(httrack -O /dev/null -#i "$1" "$2" "$3")
|
||||
test "$got" == "$4" ||
|
||||
{
|
||||
echo "FAIL ident($1, $2, $3): got '$got' want '$4'"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
### lienrelatif
|
||||
|
||||
rel 'dir/page.html' 'dir/index.html' 'page.html'
|
||||
rel 'dir/page.html' 'dir/page.html' 'page.html' # self-link
|
||||
rel 'a.html' 'dir/index.html' '../a.html'
|
||||
rel 'x.html' 'a/b/c/index.html' '../../../x.html'
|
||||
rel 'h/a/x.jpg' 'h/a/sub/page.html' '../x.jpg'
|
||||
rel 'a/b/c/x.html' 'index.html' 'a/b/c/x.html'
|
||||
rel 'h/sub/x.jpg' 'h/page.html' 'sub/x.jpg'
|
||||
rel 'h/dir2/x.jpg' 'h/dir1/page.html' '../dir2/x.jpg' # sibling dir
|
||||
rel 'h/bc/x.jpg' 'h/b/page.html' '../bc/x.jpg' # b/bc prefix trap
|
||||
rel 'h/b/x.jpg' 'h/bc/page.html' '../b/x.jpg'
|
||||
rel 'h2/img/x.jpg' 'h1/p/page.html' '../../h2/img/x.jpg' # cross-host
|
||||
rel 'img.cdn/photo.jpg' 'www.site/articles/2020/post.html' '../../../img.cdn/photo.jpg'
|
||||
rel 'h/a/' 'h/a/sub/page.html' '../' # link is ancestor dir
|
||||
rel 'x.html' 'page.html' 'x.html'
|
||||
rel 'dir/page.html?x=1' 'dir/index.html?y=2' 'page.html' # ? stripped
|
||||
|
||||
### ident_url_relatif
|
||||
|
||||
ident 'img.gif' 'www.foo.com' '/dir/page.html' 'adr=www.foo.com fil=/dir/img.gif'
|
||||
ident 'sub/img.gif' 'www.foo.com' '/dir/page.html' 'adr=www.foo.com fil=/dir/sub/img.gif'
|
||||
ident '/img.gif' 'www.foo.com' '/dir/page.html' 'adr=www.foo.com fil=/img.gif'
|
||||
# embedded ../ collapses (#137)
|
||||
ident '../img.gif' 'www.foo.com' '/dir/sub/page.html' 'adr=www.foo.com fil=/dir/img.gif'
|
||||
ident 'sub/../logo.png' 'www.foo.com' '/articles/2020/post.html' 'adr=www.foo.com fil=/articles/2020/logo.png'
|
||||
ident '../../pix/sub/../logo.png' 'www.foo.com' '/articles/2020/post.html' 'adr=www.foo.com fil=/pix/logo.png'
|
||||
ident '../../../../x.gif' 'www.foo.com' '/a/b/page.html' 'adr=www.foo.com fil=/x.gif' # above-root clamp
|
||||
ident '?page=2' 'www.foo.com' '/dir/index.html?old=1' 'adr=www.foo.com fil=/dir/index.html?page=2'
|
||||
ident 'http://other.com/a/b/../c/index.html' 'www.foo.com' '/p.html' 'adr=other.com fil=/a/c/index.html'
|
||||
# file:// collapses ../ like the other schemes; traversal contained, // authority kept
|
||||
ident 'file:///var/data/pix/sub/../logo.png' 'www.foo.com' '/p.html' 'adr=file:// fil=/var/data/pix/logo.png'
|
||||
ident 'file:///a/b/c/../../d/e.gif' 'www.foo.com' '/p.html' 'adr=file:// fil=/a/d/e.gif'
|
||||
ident 'file:///a/../../b' 'www.foo.com' '/p.html' 'adr=file:// fil=/b'
|
||||
ident 'file://srv/share/../x' 'www.foo.com' '/p.html' 'adr=file:// fil=//srv/x'
|
||||
ident 'mailto:foo@bar.com' 'www.foo.com' '/p.html' 'error=-1' # unsupported scheme
|
||||
ident 'javascript:void(0)' 'www.foo.com' '/p.html' 'error=-1'
|
||||
|
||||
echo "OK"
|
||||
@@ -5,7 +5,7 @@ set -euo pipefail
|
||||
|
||||
# path simplify engine (fil_simplifie): collapses ./ and ../ segments.
|
||||
simp() {
|
||||
test "$(httrack -O /dev/null -#1 "$1")" == "simplified=$2" || exit 1
|
||||
test "$(httrack -O /dev/null -#1 "$1")" == "simplified=$2" || exit 1
|
||||
}
|
||||
|
||||
simp './foo/bar/' 'foo/bar/'
|
||||
@@ -26,3 +26,17 @@ simp './a/../../b' 'b'
|
||||
|
||||
# empty segments ('//') are not dot-segments and are preserved, per RFC 3986
|
||||
simp 'a//b' 'a//b'
|
||||
simp 'a//b/../c' 'a//c'
|
||||
|
||||
# absolute paths keep the leading '/'; above-root '..' is clamped to it
|
||||
simp '/a/../b' '/b'
|
||||
simp '/a/../../b' '/b'
|
||||
simp '/../x' '/x'
|
||||
|
||||
# collapses to nothing -> './' (relative) or '/' (absolute)
|
||||
simp '..' './'
|
||||
simp 'a/..' './'
|
||||
simp '/' '/'
|
||||
|
||||
simp 'a/b/..' 'a/' # trailing bare '..'
|
||||
simp 'a/../b?x=../y' 'b?x=../y' # '?' freezes simplification
|
||||
|
||||
@@ -21,9 +21,15 @@ test "$out" == "strsafe: OK" || exit 1
|
||||
# the bounded macro aborts (non-zero exit), so don't let set -e trip on it
|
||||
err=$(httrack -#8 overflow "this string is far too long for the buffer" 2>&1) || true
|
||||
case "$err" in
|
||||
*"strsafe: NOT aborted"*) echo "over-capacity write was NOT caught" >&2; exit 1 ;;
|
||||
*"overflow while copying"*) ;;
|
||||
*) echo "expected htssafe overflow abort, got: $err" >&2; exit 1 ;;
|
||||
*"strsafe: NOT aborted"*)
|
||||
echo "over-capacity write was NOT caught" >&2
|
||||
exit 1
|
||||
;;
|
||||
*"overflow while copying"*) ;;
|
||||
*)
|
||||
echo "expected htssafe overflow abort, got: $err" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Same guarantee for the htsbuff builder. The source is exactly the buffer
|
||||
@@ -32,7 +38,13 @@ esac
|
||||
# aborted"). Match the specific htsbuff abort message, not just any assert.
|
||||
err=$(httrack -#8 overflow-buff "abcd" 2>&1) || true
|
||||
case "$err" in
|
||||
*"strsafe: NOT aborted"*) echo "htsbuff over-capacity write was NOT caught" >&2; exit 1 ;;
|
||||
*"htsbuff append overflow"*) ;;
|
||||
*) echo "expected htsbuff overflow abort, got: $err" >&2; exit 1 ;;
|
||||
*"strsafe: NOT aborted"*)
|
||||
echo "htsbuff over-capacity write was NOT caught" >&2
|
||||
exit 1
|
||||
;;
|
||||
*"htsbuff append overflow"*) ;;
|
||||
*)
|
||||
echo "expected htsbuff overflow abort, got: $err" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
|
||||
bash crawl-test.sh --errors 0 --files 5 httrack http://ut.httrack.com/simple/basic.html
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
|
||||
bash crawl-test.sh --errors 0 --files 3 \
|
||||
--found ut.httrack.com/cookies/third.html \
|
||||
--found ut.httrack.com/cookies/second.html \
|
||||
--found ut.httrack.com/cookies/entrance.html \
|
||||
httrack http://ut.httrack.com/cookies/entrance.php
|
||||
--found ut.httrack.com/cookies/third.html \
|
||||
--found ut.httrack.com/cookies/second.html \
|
||||
--found ut.httrack.com/cookies/entrance.html \
|
||||
httrack http://ut.httrack.com/cookies/entrance.php
|
||||
|
||||
@@ -3,21 +3,21 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
|
||||
# unicode tests
|
||||
bash crawl-test.sh \
|
||||
--errors 1 --files 5 \
|
||||
--found 'café.ut.httrack.com/unicode-links/café3860.html' \
|
||||
--found 'café.ut.httrack.com/unicode-links/café30f4.html' \
|
||||
--found 'café.ut.httrack.com/unicode-links/café5e1f.html' \
|
||||
--found 'café.ut.httrack.com/unicode-links/café7b30.html' \
|
||||
httrack 'http://ut.httrack.com/unicode-links/idna.html' \
|
||||
'+*.ut.httrack.com/*' --robots=0
|
||||
--errors 1 --files 5 \
|
||||
--found 'café.ut.httrack.com/unicode-links/café3860.html' \
|
||||
--found 'café.ut.httrack.com/unicode-links/café30f4.html' \
|
||||
--found 'café.ut.httrack.com/unicode-links/café5e1f.html' \
|
||||
--found 'café.ut.httrack.com/unicode-links/café7b30.html' \
|
||||
httrack 'http://ut.httrack.com/unicode-links/idna.html' \
|
||||
'+*.ut.httrack.com/*' --robots=0
|
||||
|
||||
# unicode tests (bogus links)
|
||||
bash crawl-test.sh \
|
||||
--errors 0 --files 1 \
|
||||
--found 'ut.httrack.com/unicode-links/idna_bogus.html' \
|
||||
httrack 'http://ut.httrack.com/unicode-links/idna_bogus.html' \
|
||||
'-*' --robots=0
|
||||
--errors 0 --files 1 \
|
||||
--found 'ut.httrack.com/unicode-links/idna_bogus.html' \
|
||||
httrack 'http://ut.httrack.com/unicode-links/idna_bogus.html' \
|
||||
'-*' --robots=0
|
||||
|
||||
@@ -3,67 +3,67 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
|
||||
# unicode tests
|
||||
bash crawl-test.sh \
|
||||
--errors 1 --files 10 \
|
||||
--found ut.httrack.com/unicode-links/caf%a91bce.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/café463e.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/café9fa8.html \
|
||||
--found ut.httrack.com/unicode-links/caféae52.html \
|
||||
--found ut.httrack.com/unicode-links/caféc009.html \
|
||||
--found ut.httrack.com/unicode-links/utf8.html \
|
||||
httrack http://ut.httrack.com/unicode-links/utf8.html
|
||||
--errors 1 --files 10 \
|
||||
--found ut.httrack.com/unicode-links/caf%a91bce.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/café463e.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/café9fa8.html \
|
||||
--found ut.httrack.com/unicode-links/caféae52.html \
|
||||
--found ut.httrack.com/unicode-links/caféc009.html \
|
||||
--found ut.httrack.com/unicode-links/utf8.html \
|
||||
httrack http://ut.httrack.com/unicode-links/utf8.html
|
||||
|
||||
bash crawl-test.sh \
|
||||
--errors 4 --files 7 \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/café9fa8.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/caf%e939bd.html \
|
||||
--found ut.httrack.com/unicode-links/caf%e9ae52.html \
|
||||
--found ut.httrack.com/unicode-links/caféaec2.html \
|
||||
--found ut.httrack.com/unicode-links/caféfad6.html \
|
||||
--found ut.httrack.com/unicode-links/default.html \
|
||||
httrack http://ut.httrack.com/unicode-links/default.html
|
||||
--errors 4 --files 7 \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/café9fa8.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/caf%e939bd.html \
|
||||
--found ut.httrack.com/unicode-links/caf%e9ae52.html \
|
||||
--found ut.httrack.com/unicode-links/caféaec2.html \
|
||||
--found ut.httrack.com/unicode-links/caféfad6.html \
|
||||
--found ut.httrack.com/unicode-links/default.html \
|
||||
httrack http://ut.httrack.com/unicode-links/default.html
|
||||
|
||||
bash crawl-test.sh \
|
||||
--errors 2 --files 9 \
|
||||
--found ut.httrack.com/unicode-links/caf%a9ae52.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a9bf59.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café647f.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/caféaec2.html \
|
||||
--found ut.httrack.com/unicode-links/caféfad6.html \
|
||||
--found ut.httrack.com/unicode-links/iso88591.html \
|
||||
httrack http://ut.httrack.com/unicode-links/iso88591.html
|
||||
--errors 2 --files 9 \
|
||||
--found ut.httrack.com/unicode-links/caf%a9ae52.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a9bf59.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café647f.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/caféaec2.html \
|
||||
--found ut.httrack.com/unicode-links/caféfad6.html \
|
||||
--found ut.httrack.com/unicode-links/iso88591.html \
|
||||
httrack http://ut.httrack.com/unicode-links/iso88591.html
|
||||
|
||||
bash crawl-test.sh \
|
||||
--errors 4 --files 9 \
|
||||
--found ut.httrack.com/unicode-links/caf%a8%a6c72a.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a9bf59.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/cafébf43.html \
|
||||
--found ut.httrack.com/unicode-links/cafédcd8.html \
|
||||
--found ut.httrack.com/unicode-links/café2461.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a8%a61bce.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a9ae52.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/gb18030.html \
|
||||
httrack http://ut.httrack.com/unicode-links/gb18030.html
|
||||
--errors 4 --files 9 \
|
||||
--found ut.httrack.com/unicode-links/caf%a8%a6c72a.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a9bf59.html \
|
||||
--found ut.httrack.com/unicode-links/café8007.html \
|
||||
--found ut.httrack.com/unicode-links/cafébf43.html \
|
||||
--found ut.httrack.com/unicode-links/cafédcd8.html \
|
||||
--found ut.httrack.com/unicode-links/café2461.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a8%a61bce.html \
|
||||
--found ut.httrack.com/unicode-links/caf%a9ae52.html \
|
||||
--found ut.httrack.com/unicode-links/café7b30.html \
|
||||
--found ut.httrack.com/unicode-links/café30f4.html \
|
||||
--found ut.httrack.com/unicode-links/café5e1f.html \
|
||||
--found ut.httrack.com/unicode-links/café3860.html \
|
||||
--found ut.httrack.com/unicode-links/gb18030.html \
|
||||
httrack http://ut.httrack.com/unicode-links/gb18030.html
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
|
||||
# http://code.google.com/p/httrack/issues/detail?id=42&can=1
|
||||
# we expect 2 errors only because other links are too longs (to be modified if suitable)
|
||||
bash crawl-test.sh --errors 2 --files 1 \
|
||||
--found ut.httrack.com/overflow/longquerywithaccents.html \
|
||||
httrack http://ut.httrack.com/overflow/longquerywithaccents.php
|
||||
--found ut.httrack.com/overflow/longquerywithaccents.html \
|
||||
httrack http://ut.httrack.com/overflow/longquerywithaccents.php
|
||||
|
||||
@@ -3,45 +3,45 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
|
||||
# http://code.google.com/p/httrack/issues/detail?id=4&can=1
|
||||
bash crawl-test.sh --errors 0 --files 4 \
|
||||
--found ut.httrack.com/parsing/back5e1f.gif \
|
||||
--found ut.httrack.com/parsing/events.html \
|
||||
--found ut.httrack.com/parsing/fade230f4.gif \
|
||||
--found ut.httrack.com/parsing/fade3860.gif \
|
||||
httrack http://ut.httrack.com/parsing/events.html
|
||||
--found ut.httrack.com/parsing/back5e1f.gif \
|
||||
--found ut.httrack.com/parsing/events.html \
|
||||
--found ut.httrack.com/parsing/fade230f4.gif \
|
||||
--found ut.httrack.com/parsing/fade3860.gif \
|
||||
httrack http://ut.httrack.com/parsing/events.html
|
||||
|
||||
# http://code.google.com/p/httrack/issues/detail?id=2&can=1
|
||||
bash crawl-test.sh --errors 0 --files 3 \
|
||||
--found ut.httrack.com/parsing/background-image.css \
|
||||
--found ut.httrack.com/parsing/background-image.html \
|
||||
--found ut.httrack.com/parsing/fade.gif \
|
||||
httrack http://ut.httrack.com/parsing/background-image.html
|
||||
--found ut.httrack.com/parsing/background-image.css \
|
||||
--found ut.httrack.com/parsing/background-image.html \
|
||||
--found ut.httrack.com/parsing/fade.gif \
|
||||
httrack http://ut.httrack.com/parsing/background-image.html
|
||||
|
||||
# javascript parsing
|
||||
bash crawl-test.sh --errors 0 --files 3 \
|
||||
--found ut.httrack.com/parsing/back.gif \
|
||||
--found ut.httrack.com/parsing/fade.gif \
|
||||
--found ut.httrack.com/parsing/javascript.html \
|
||||
httrack http://ut.httrack.com/parsing/javascript.html
|
||||
--found ut.httrack.com/parsing/back.gif \
|
||||
--found ut.httrack.com/parsing/fade.gif \
|
||||
--found ut.httrack.com/parsing/javascript.html \
|
||||
httrack http://ut.httrack.com/parsing/javascript.html
|
||||
|
||||
# handling of + before query string
|
||||
bash crawl-test.sh --errors 0 --files 6 \
|
||||
--found ut.httrack.com/parsing/escaping.html \
|
||||
--found "ut.httrack.com/parsing/foo bar30f4.html" \
|
||||
--found "ut.httrack.com/parsing/foo bar5e1f.html" \
|
||||
--found "ut.httrack.com/parsing/foo+bar+plus3860.html" \
|
||||
--found "ut.httrack.com/parsing/foo barae52.html" \
|
||||
--found "ut.httrack.com/parsing/foo bar7b30.html" \
|
||||
httrack http://ut.httrack.com/parsing/escaping.html
|
||||
--found ut.httrack.com/parsing/escaping.html \
|
||||
--found "ut.httrack.com/parsing/foo bar30f4.html" \
|
||||
--found "ut.httrack.com/parsing/foo bar5e1f.html" \
|
||||
--found "ut.httrack.com/parsing/foo+bar+plus3860.html" \
|
||||
--found "ut.httrack.com/parsing/foo barae52.html" \
|
||||
--found "ut.httrack.com/parsing/foo bar7b30.html" \
|
||||
httrack http://ut.httrack.com/parsing/escaping.html
|
||||
|
||||
# handling of # encoded in filename
|
||||
# see http://code.google.com/p/httrack/issues/detail?id=25
|
||||
bash crawl-test.sh --errors 2 --files 4 \
|
||||
--found "ut.httrack.com/parsing/escaping2.html" \
|
||||
--found "ut.httrack.com/parsing/++foo++bar++plus++.html" \
|
||||
--found "ut.httrack.com/parsing/foo#bar#.html" \
|
||||
--found "ut.httrack.com/parsing/foo bar.html" \
|
||||
httrack http://ut.httrack.com/parsing/escaping2.html
|
||||
--found "ut.httrack.com/parsing/escaping2.html" \
|
||||
--found "ut.httrack.com/parsing/++foo++bar++plus++.html" \
|
||||
--found "ut.httrack.com/parsing/foo#bar#.html" \
|
||||
--found "ut.httrack.com/parsing/foo bar.html" \
|
||||
httrack http://ut.httrack.com/parsing/escaping2.html
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
bash check-network.sh || ! echo "skipping online unit tests" || exit 77
|
||||
|
||||
if test "${HTTPS_SUPPORT:-}" == "no"; then
|
||||
echo "no https support compiled, skipping"
|
||||
exit 77
|
||||
echo "no https support compiled, skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
bash crawl-test.sh --errors 0 --files 5 httrack https://ut.httrack.com/simple/basic.html
|
||||
|
||||
@@ -35,6 +35,7 @@ TESTS = \
|
||||
01_engine-mime.test \
|
||||
01_engine-parse.test \
|
||||
01_engine-rcfile.test \
|
||||
01_engine-relative.test \
|
||||
01_engine-simplify.test \
|
||||
01_engine-strsafe.test \
|
||||
02_manpage-regen.test \
|
||||
|
||||
@@ -6,39 +6,39 @@
|
||||
|
||||
# do not enable online tests (./configure --disable-online-unit-tests)
|
||||
if test "$ONLINE_UNIT_TESTS" == "no"; then
|
||||
echo "online tests are disabled" >&2
|
||||
exit 1
|
||||
echo "online tests are disabled" >&2
|
||||
exit 1
|
||||
|
||||
# enable online tests (--enable-online-unit-tests)
|
||||
elif test "$ONLINE_UNIT_TESTS" == "yes"; then
|
||||
exit 0
|
||||
exit 0
|
||||
|
||||
# check if online tests are reachable
|
||||
else
|
||||
|
||||
# test url
|
||||
url=http://ut.httrack.com/enabled
|
||||
# test url
|
||||
url=http://ut.httrack.com/enabled
|
||||
|
||||
# cache file name
|
||||
cache=check-network_sh.cache
|
||||
# cache file name
|
||||
cache=check-network_sh.cache
|
||||
|
||||
# cached result ?
|
||||
if test -f $cache ; then
|
||||
if grep -q "ok" $cache ; then
|
||||
exit 0
|
||||
else
|
||||
echo "online tests are disabled (cached)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# cached result ?
|
||||
if test -f $cache; then
|
||||
if grep -q "ok" $cache; then
|
||||
exit 0
|
||||
else
|
||||
echo "online tests are disabled (cached)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# fetch single file
|
||||
elif bash crawl-test.sh --errors 0 --files 1 httrack --timeout=3 --max-time=3 "$url" 2>/dev/null >/dev/null ; then
|
||||
echo "ok" > $cache
|
||||
exit 0
|
||||
else
|
||||
echo "error" > $cache
|
||||
echo "online tests are disabled (auto)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# fetch single file
|
||||
elif bash crawl-test.sh --errors 0 --files 1 httrack --timeout=3 --max-time=3 "$url" 2>/dev/null >/dev/null; then
|
||||
echo "ok" >$cache
|
||||
exit 0
|
||||
else
|
||||
echo "error" >$cache
|
||||
echo "online tests are disabled (auto)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
@@ -2,185 +2,184 @@
|
||||
#
|
||||
|
||||
function warning {
|
||||
echo "** $*" >&2
|
||||
return 0
|
||||
echo "** $*" >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
function die {
|
||||
warning "$*"
|
||||
exit 1
|
||||
warning "$*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
function debug {
|
||||
if test -n "$verbose"; then
|
||||
echo "$*" >&2
|
||||
fi
|
||||
if test -n "$verbose"; then
|
||||
echo "$*" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
function info {
|
||||
printf "[$*] ..\t" >&2
|
||||
printf '[%s] ..\t' "$*" >&2
|
||||
}
|
||||
|
||||
function result {
|
||||
echo "$*" >&2
|
||||
echo "$*" >&2
|
||||
}
|
||||
|
||||
function cleanup {
|
||||
debug "cleaning function called"
|
||||
if test -n "$tmpdir"; then
|
||||
if test -d "$tmpdir"; then
|
||||
if test -z "$nopurge"; then
|
||||
debug "cleaning up $tmpdir"
|
||||
rm -rf "$tmpdir"
|
||||
fi
|
||||
debug "cleaning function called"
|
||||
if test -n "$tmpdir"; then
|
||||
if test -d "$tmpdir"; then
|
||||
if test -z "$nopurge"; then
|
||||
debug "cleaning up $tmpdir"
|
||||
rm -rf "$tmpdir"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if test -n "$crawlpid"; then
|
||||
debug "killing $crawlpid"
|
||||
kill -9 "$crawlpid"
|
||||
crawlpid=
|
||||
fi
|
||||
fi
|
||||
if test -n "$crawlpid"; then
|
||||
debug "killing $crawlpid"
|
||||
kill -9 "$crawlpid"
|
||||
crawlpid=
|
||||
fi
|
||||
}
|
||||
|
||||
function usage {
|
||||
cat << EOF
|
||||
cat <<EOF
|
||||
usage: $0
|
||||
EOF
|
||||
}
|
||||
|
||||
function assert_equals {
|
||||
info "$1"
|
||||
if test ! "$2" == "$3"; then
|
||||
result "expected '$2', got '$3'"
|
||||
exit 1
|
||||
else
|
||||
result "OK ($2)"
|
||||
fi
|
||||
info "$1"
|
||||
if test ! "$2" == "$3"; then
|
||||
result "expected '$2', got '$3'"
|
||||
exit 1
|
||||
else
|
||||
result "OK ($2)"
|
||||
fi
|
||||
}
|
||||
|
||||
function start-crawl {
|
||||
# parse args
|
||||
pos=1
|
||||
while test "$#" -ge "$pos" ; do
|
||||
case "${!pos}" in
|
||||
--debug)
|
||||
verbose=1
|
||||
;;
|
||||
--no-purge|--summary|--print-files)
|
||||
;;
|
||||
--errors|--files|--found|--not-found|--directory)
|
||||
pos=$[${pos}+1]
|
||||
test "$#" -ge "$pos" || warning "missing argument" || return 1
|
||||
;;
|
||||
httrack)
|
||||
pos=$[${pos}+1]
|
||||
break;
|
||||
;;
|
||||
*)
|
||||
warning "unrecognized option ${!pos}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
pos=$[${pos}+1]
|
||||
done
|
||||
debug "remaining args: ${@:${pos}}"
|
||||
# parse args
|
||||
pos=1
|
||||
while test "$#" -ge "$pos"; do
|
||||
case "${!pos}" in
|
||||
--debug)
|
||||
verbose=1
|
||||
;;
|
||||
--no-purge | --summary | --print-files) ;;
|
||||
--errors | --files | --found | --not-found | --directory)
|
||||
pos=$((pos + 1))
|
||||
test "$#" -ge "$pos" || warning "missing argument" || return 1
|
||||
;;
|
||||
httrack)
|
||||
pos=$((pos + 1))
|
||||
break
|
||||
;;
|
||||
*)
|
||||
warning "unrecognized option ${!pos}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
pos=$((pos + 1))
|
||||
done
|
||||
debug "remaining args: ${*:pos}"
|
||||
|
||||
# ut/ won't exceed 2 minutes
|
||||
moreargs="--quiet --max-time=120 --timeout=30 --connection-per-second=5"
|
||||
# ut/ won't exceed 2 minutes
|
||||
moreargs=(--quiet --max-time=120 --timeout=30 --connection-per-second=5)
|
||||
|
||||
# proxy environment ?
|
||||
if test -n "$http_proxy"; then
|
||||
moreargs="$moreargs --proxy $http_proxy"
|
||||
fi
|
||||
# proxy environment ?
|
||||
if test -n "${http_proxy:-}"; then
|
||||
moreargs+=(--proxy "$http_proxy")
|
||||
fi
|
||||
|
||||
test -n "$tmpdir" || ! warning "no tmpdir" || return 1
|
||||
tmp="${tmpdir}/crawl"
|
||||
rm -rf "$tmp"
|
||||
mkdir "$tmp" || ! warning "could not create $tmp" || return 1
|
||||
|
||||
which httrack >/dev/null || ! warning "could not find httrack" || return 1
|
||||
ver=$(httrack -O /dev/null --version | sed -e 's/HTTrack version //')
|
||||
test -n "$ver" || ! warning "could not run httrack" || return 1
|
||||
|
||||
# start crawl
|
||||
log="${tmp}/log"
|
||||
debug starting httrack -O "${tmp}" ${moreargs} ${@:${pos}}
|
||||
info "running httrack ${@:${pos}}"
|
||||
httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" ${moreargs} ${@:${pos}} >"${log}" 2>&1 &
|
||||
crawlpid="$!"
|
||||
debug "started cralwer on pid $crawlpid"
|
||||
wait "$crawlpid"
|
||||
result="$?"
|
||||
crawlpid=
|
||||
test "$result" -eq 0 || ! result "error code $result" || return 1
|
||||
result "OK"
|
||||
grep -iE "^[0-9\:]*[[:space:]]Error:" "${tmp}/hts-log.txt" >&2
|
||||
|
||||
# now audit
|
||||
while test "$#" -gt 0; do
|
||||
case "$1" in
|
||||
--no-purge)
|
||||
nopurge=1
|
||||
;;
|
||||
--summary)
|
||||
grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${tmp}/hts-log.txt"
|
||||
;;
|
||||
--print-files)
|
||||
find "${tmp}" -mindepth 1 -type f
|
||||
;;
|
||||
--errors)
|
||||
shift
|
||||
assert_equals "checking errors" "$1" "$(grep -iEc "^[0-9\:]*[[:space:]]Error:" "${tmp}/hts-log.txt")"
|
||||
;;
|
||||
--found)
|
||||
shift
|
||||
info "checking for $1"
|
||||
if test -f "${tmp}/$1" ; then
|
||||
result "OK"
|
||||
else
|
||||
result "not found"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--not-found)
|
||||
shift
|
||||
info "checking for $1"
|
||||
if test -f "${tmp}/$1" ; then
|
||||
result "OK"
|
||||
else
|
||||
result "not found"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--directory)
|
||||
shift
|
||||
info "checking for $1"
|
||||
if test -d "${tmp}/$1" ; then
|
||||
result "OK"
|
||||
else
|
||||
result "not found"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--files)
|
||||
shift
|
||||
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${tmp}/hts-log.txt" \
|
||||
| sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
|
||||
assert_equals "checking files" "$1" "$nFiles"
|
||||
;;
|
||||
httrack)
|
||||
break;
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# cleanup
|
||||
if test -z "$nopurge"; then
|
||||
test -n "$tmpdir" || ! warning "no tmpdir" || return 1
|
||||
tmp="${tmpdir}/crawl"
|
||||
rm -rf "$tmp"
|
||||
else
|
||||
tmpdir=
|
||||
fi
|
||||
mkdir "$tmp" || ! warning "could not create $tmp" || return 1
|
||||
|
||||
which httrack >/dev/null || ! warning "could not find httrack" || return 1
|
||||
ver=$(httrack -O /dev/null --version | sed -e 's/HTTrack version //')
|
||||
test -n "$ver" || ! warning "could not run httrack" || return 1
|
||||
|
||||
# start crawl
|
||||
log="${tmp}/log"
|
||||
debug starting httrack -O "${tmp}" "${moreargs[@]}" "${@:pos}"
|
||||
info "running httrack ${*:pos}"
|
||||
httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
|
||||
crawlpid="$!"
|
||||
debug "started cralwer on pid $crawlpid"
|
||||
wait "$crawlpid"
|
||||
result="$?"
|
||||
crawlpid=
|
||||
test "$result" -eq 0 || ! result "error code $result" || return 1
|
||||
result "OK"
|
||||
grep -iE "^[0-9\:]*[[:space:]]Error:" "${tmp}/hts-log.txt" >&2
|
||||
|
||||
# now audit
|
||||
while test "$#" -gt 0; do
|
||||
case "$1" in
|
||||
--no-purge)
|
||||
nopurge=1
|
||||
;;
|
||||
--summary)
|
||||
grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${tmp}/hts-log.txt"
|
||||
;;
|
||||
--print-files)
|
||||
find "${tmp}" -mindepth 1 -type f
|
||||
;;
|
||||
--errors)
|
||||
shift
|
||||
assert_equals "checking errors" "$1" "$(grep -iEc "^[0-9\:]*[[:space:]]Error:" "${tmp}/hts-log.txt")"
|
||||
;;
|
||||
--found)
|
||||
shift
|
||||
info "checking for $1"
|
||||
if test -f "${tmp}/$1"; then
|
||||
result "OK"
|
||||
else
|
||||
result "not found"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--not-found)
|
||||
shift
|
||||
info "checking for $1"
|
||||
if test -f "${tmp}/$1"; then
|
||||
result "OK"
|
||||
else
|
||||
result "not found"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--directory)
|
||||
shift
|
||||
info "checking for $1"
|
||||
if test -d "${tmp}/$1"; then
|
||||
result "OK"
|
||||
else
|
||||
result "not found"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--files)
|
||||
shift
|
||||
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${tmp}/hts-log.txt" |
|
||||
sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
|
||||
assert_equals "checking files" "$1" "$nFiles"
|
||||
;;
|
||||
httrack)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# cleanup
|
||||
if test -z "$nopurge"; then
|
||||
rm -rf "$tmp"
|
||||
else
|
||||
tmpdir=
|
||||
fi
|
||||
}
|
||||
|
||||
# check args
|
||||
@@ -195,7 +194,7 @@ tmpdir=
|
||||
crawlpid=
|
||||
nopurge=
|
||||
verbose=
|
||||
trap "cleanup" 0 1 2 3 4 5 6 7 8 9 11 13 14 15 16 19 24 25
|
||||
trap cleanup EXIT HUP INT QUIT ILL TRAP ABRT BUS FPE SEGV PIPE ALRM TERM STKFLT XCPU XFSZ
|
||||
|
||||
# working directory
|
||||
tmpdir="${tmptopdir}/httrack_ut.$$"
|
||||
|
||||
@@ -2,19 +2,19 @@
|
||||
#
|
||||
|
||||
error=0
|
||||
for i in *.test ; do
|
||||
if bash $i ; then
|
||||
echo "$i: passed" >&2
|
||||
else
|
||||
echo "$i: ERROR" >&2
|
||||
error=$[${error}+1]
|
||||
fi
|
||||
for i in *.test; do
|
||||
if bash "$i"; then
|
||||
echo "$i: passed" >&2
|
||||
else
|
||||
echo "$i: ERROR" >&2
|
||||
error=$((error + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
if test "$error" -eq 0; then
|
||||
echo "all tests passed" >&2
|
||||
echo "all tests passed" >&2
|
||||
else
|
||||
echo "${error} test(s) failed" >&2
|
||||
echo "${error} test(s) failed" >&2
|
||||
fi
|
||||
|
||||
exit $error
|
||||
|
||||
Reference in New Issue
Block a user