mirror of
https://github.com/xroche/httrack.git
synced 2026-07-26 18:49:39 +03:00
Compare commits
2 Commits
feat/singl
...
warnfix-co
| Author | SHA1 | Date | |
|---|---|---|---|
|
8188b55c84 | ||
|
|
c169db5ab9 |
5
.github/workflows/windows-build.yml
vendored
5
.github/workflows/windows-build.yml
vendored
@@ -225,9 +225,8 @@ jobs:
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
|
||||
# webdav-mime needs a reapable background listener, which MSYS cannot give it.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test"
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test"
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
@@ -100,8 +100,7 @@ AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLA
|
||||
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"], [], [-Werror])], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"], [], [-Werror])
|
||||
# No --discard-all: it drops the local symbols naming every static function, so
|
||||
# a trace misattributes them to the nearest surviving global. Costs 0.6% size.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--no-undefined"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
|
||||
@@ -128,8 +127,8 @@ AX_CHECK_LINK_FLAG([-pie], [LDFLAGS_PIE="-pie"])
|
||||
AC_SUBST([CFLAGS_PIE])
|
||||
AC_SUBST([LDFLAGS_PIE])
|
||||
|
||||
# Ties a crash trace from a stripped build back to its separate debug symbols.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--build-id], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--build-id"])
|
||||
## -rdynamic must be a link flag; DEFAULT_CFLAGS (AM_CPPFLAGS) never reaches the linker.
|
||||
AX_CHECK_LINK_FLAG([-rdynamic], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -rdynamic"])
|
||||
|
||||
### Check for -fvisibility=hidden support
|
||||
gl_VISIBILITY
|
||||
|
||||
@@ -300,7 +300,6 @@ ones it kept so the local copy browses offline. These options tune both halves.<
|
||||
<tr><td><tt>--preserve (-%p), --disable-passwords (-%x)</tt></td><td>Leave HTML untouched (no rewriting), and strip passwords out of saved links.</td></tr>
|
||||
<tr><td><tt>--extended-parsing (-%P), --parse-java (-j)</tt></td><td>Aggressive link discovery, and how much script content is parsed for links.</td></tr>
|
||||
<tr><td><tt>--mime-html (-%M)</tt></td><td>Save the whole mirror as a single MIME-encapsulated <tt>.mht</tt> archive (<tt>index.mht</tt>).</td></tr>
|
||||
<tr><td><tt>--single-file (-%Z), --single-file-max-size N</tt></td><td>Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded as <tt>data:</tt> URIs. Assets over the cap (10 MB by default) keep their link, as do audio, video, and the links from one page to another.</td></tr>
|
||||
<tr><td><tt>--index (-I), --build-top-index (-%i), --search-index (-%I)</tt></td><td>Build a per-mirror index, a top index across projects, and a searchable keyword index.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -483,18 +482,6 @@ add <tt>--warc-cdx</tt> for a sorted CDXJ index, or <tt>--wacz</tt> to bundle th
|
||||
archive, index and pages into one WACZ for replay tools such as
|
||||
replayweb.page.</small></p>
|
||||
|
||||
<h4>Pages you can hand to someone as one file</h4>
|
||||
<p><tt>httrack https://example.com/ --single-file --path mydir</tt><br>
|
||||
<small>Rewrites each saved page after the crawl so its stylesheets, scripts,
|
||||
images and fonts are embedded as <tt>data:</tt> URIs. The mirror stays a normal
|
||||
browsable tree — links between pages remain relative, and the assets are
|
||||
still on disk — but any single <tt>.html</tt> file can now be mailed or
|
||||
archived on its own. Unlike <tt>--mime-html</tt>, which produces one
|
||||
<tt>.mht</tt> archive that current browsers no longer open, the output is
|
||||
ordinary HTML. Raise or lower the 10 MB per-asset limit with
|
||||
<tt>--single-file-max-size N</tt>; anything over it, plus audio and video, keeps
|
||||
its link.</small></p>
|
||||
|
||||
<h4>HTTrack as a fetch tool</h4>
|
||||
<p><tt>httrack --get https://host/file.bin --path tmp</tt><br>
|
||||
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
|
||||
|
||||
@@ -90,10 +90,9 @@ offline browser : copy websites to a local directory</p>
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-%Z, --single-file</b> ] [ <b>-LN,
|
||||
--long-names[=N]</b> ] [ <b>-KN, --keep-links[=N]</b> ] [
|
||||
<b>-x, --replace-external</b> ] [ <b>-%x,
|
||||
--disable-passwords</b> ] [ <b>-%q,
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
<b>-%x, --disable-passwords</b> ] [ <b>-%q,
|
||||
--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
|
||||
<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
|
||||
] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
|
||||
@@ -649,22 +648,6 @@ don’t wait) (--cached-delayed-type-check)</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%Z</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>after the mirror, rewrite each saved page with its
|
||||
stylesheets, scripts, images and fonts inlined as data:
|
||||
URIs, so any page also opens on its own (links between pages
|
||||
stay relative; audio and video stay links);
|
||||
--single-file-max-size N caps each asset (default 10485760
|
||||
bytes) (--single-file)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%t</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
@@ -136,13 +136,6 @@ ${listid:build:LISTDEF_3}
|
||||
<tr><td><input type="checkbox" name="nopurge" ${checked:nopurge}
|
||||
title='${html:LANG_I1a}' onMouseOver="info('${html:LANG_I1a}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I57}</td></tr>
|
||||
<tr><td><input type="checkbox" name="singlefile" ${checked:singlefile}
|
||||
title='${html:LANG_SINGLEFILETIP}' onMouseOver="info('${html:LANG_SINGLEFILETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_SINGLEFILE}</td></tr>
|
||||
<tr><td>${LANG_SINGLEFILEMAX}
|
||||
<input name="singlefilemax" value="${singlefilemax}" size="12"
|
||||
title='${html:LANG_SINGLEFILEMAXTIP}' onMouseOver="info('${html:LANG_SINGLEFILEMAXTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
></td></tr>
|
||||
</table>
|
||||
|
||||
<tr><td>
|
||||
|
||||
@@ -103,7 +103,7 @@ ${LANG_Q3}
|
||||
<tr><td>
|
||||
<table width="100%">
|
||||
<tr><td align="left">
|
||||
<input type="submit" value="${LANG_OK}"
|
||||
<input type="submit" value="${LANG_OK]"
|
||||
${do:output-mode:html-urlescaped}
|
||||
onClick="if (confirm(str_replace(str_replace('${LANG_DIAL7}', '%20', ' '), '%0a', ' '))) { form.closeme.value=1; form.submit(); } return false;"
|
||||
${do:output-mode:}
|
||||
|
||||
@@ -143,8 +143,6 @@ ${do:copy:StripQuery:stripquery}
|
||||
${do:copy:StoreAllInCache:cache2}
|
||||
${do:copy:Warc:warc}
|
||||
${do:copy:WarcFile:warcfile}
|
||||
${do:copy:SingleFile:singlefile}
|
||||
${do:copy:SingleFileMaxSize:singlefilemax}
|
||||
${do:copy:LogType:logtype}
|
||||
${do:copy:UseHTTPProxyForFTP:ftpprox}
|
||||
${do:copy:ProxyType:proxytype}
|
||||
|
||||
@@ -121,9 +121,9 @@ httrack \
|
||||
--quiet \
|
||||
--build-top-index \
|
||||
${test:todo:--mirror:--mirror:--mirror-wizard:--get:--mirrorlinks:--testlinks:--continue:--update}
|
||||
${unquoted:urls}
|
||||
${test:filelist:-%L "}${arg:filelist}${test:filelist:"}
|
||||
--path "${arg:path}/${arg:projname}"
|
||||
${urls}
|
||||
${test:filelist:-%L "}${filelist}${test:filelist:"}
|
||||
--path "${html:path}/${html:projname}"
|
||||
\
|
||||
${test:parseall:--near}
|
||||
${test:link:--test}
|
||||
@@ -131,7 +131,7 @@ httrack \
|
||||
${test:htmlfirst::--priority=7}
|
||||
\
|
||||
${do:if-not-empty:BuildString}
|
||||
--structure "${arg:BuildString}"
|
||||
--structure "${BuildString}"
|
||||
${do:end-if}
|
||||
${test:build:-N0:-N0:-N1:-N2:-N3:-N4:-N5:-N100:-N101:-N102:-N103:-N104:-N105:-N99:-N199:}
|
||||
\
|
||||
@@ -150,30 +150,29 @@ ${do:end-if}
|
||||
${test:travel3::--keep-links=0:--keep-links:--keep-links=3:--keep-links=4}
|
||||
${test:windebug:--debug-headers}
|
||||
\
|
||||
${test:connexion:--sockets=}${unquoted:connexion}
|
||||
${test:connexion:--sockets=}${connexion}
|
||||
${test:ka:--keep-alive}
|
||||
${test:timeout:--timeout=}${unquoted:timeout}
|
||||
${test:timeout:--timeout=}${timeout}
|
||||
${test:remt:--host-control=1}
|
||||
${test:retry:--retries=}${unquoted:retry}
|
||||
${test:rate:--min-rate=}${unquoted:rate}
|
||||
${test:retry:--retries=}${retry}
|
||||
${test:rate:--min-rate=}${rate}
|
||||
${test:rems:--host-control=2}
|
||||
\
|
||||
${test:depth:--depth=}${unquoted:depth}
|
||||
${test:depth2:--ext-depth=}${unquoted:depth2}
|
||||
${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one */}
|
||||
${test:othermax:--max-files=}${unquoted:othermax}
|
||||
${test:maxhtml:--max-files=,}${unquoted:maxhtml}
|
||||
${test:sizemax:--max-size=}${unquoted:sizemax}
|
||||
${test:pausebytes:--max-pause=}${unquoted:pausebytes}
|
||||
${test:maxtime:--max-time=}${unquoted:maxtime}
|
||||
${test:maxrate:--max-rate=}${unquoted:maxrate}
|
||||
${test:maxconn:--connection-per-second=}${unquoted:maxconn}
|
||||
${test:maxlinks:--advanced-maxlinks=}${unquoted:maxlinks}
|
||||
${test:depth:--depth=}${depth}
|
||||
${test:depth2:--ext-depth=}${depth2}
|
||||
${test:maxhtml:--max-files=,}${maxhtml}
|
||||
${test:othermax:--max-files=}${othermax}
|
||||
${test:sizemax:--max-files=}${sizemax}
|
||||
${test:pausebytes:--max-pause=}${pausebytes}
|
||||
${test:maxtime:--max-time=}${maxtime}
|
||||
${test:maxrate:--max-rate=}${maxrate}
|
||||
${test:maxconn:--connection-per-second=}${maxconn}
|
||||
${test:maxlinks:--advanced-maxlinks=}${maxlinks}
|
||||
\
|
||||
--user-agent "${arg:user}"
|
||||
--footer "${arg:footer}"
|
||||
--user-agent "${html:user}"
|
||||
--footer "${html:footer}"
|
||||
\
|
||||
${unquoted:url2}
|
||||
${url2}
|
||||
\
|
||||
${test:cookies:--cookies=0:}
|
||||
${test:parsejava:--parse-java=0:}
|
||||
@@ -182,22 +181,20 @@ ${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one *
|
||||
${test:keepwww:--keep-www-prefix}
|
||||
${test:keepslashes:--keep-double-slashes}
|
||||
${test:keepqueryorder:--keep-query-order}
|
||||
${test:cookiesfile:--cookies-file "}${arg:cookiesfile}${test:cookiesfile:"}
|
||||
${test:pausefiles:--pause "}${arg:pausefiles}${test:pausefiles:"}
|
||||
${test:stripquery:--strip-query "}${arg:stripquery}${test:stripquery:"}
|
||||
${test:cookiesfile:--cookies-file "}${html:cookiesfile}${test:cookiesfile:"}
|
||||
${test:pausefiles:--pause "}${pausefiles}${test:pausefiles:"}
|
||||
${test:stripquery:--strip-query "}${html:stripquery}${test:stripquery:"}
|
||||
${test:toler:--tolerant}
|
||||
${test:http10:--http-10}
|
||||
${test:cache2:--store-all-in-cache}
|
||||
${test:warc:--warc}
|
||||
${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
|
||||
${test:singlefile:--single-file}
|
||||
${test:singlefilemax:--single-file-max-size=}${unquoted:singlefilemax}
|
||||
${test:warcfile:--warc-file "}${html:warcfile}${test:warcfile:"}
|
||||
${test:norecatch:--do-not-recatch}
|
||||
${test:logf:--single-log}
|
||||
${test:logtype:::--extra-log:--debug-log}
|
||||
${test:index:--index=0:}
|
||||
${test:index2:--search-index=0:--search-index}
|
||||
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${arg:prox}${test:prox:\3A}${arg:portprox}${test:prox:"}
|
||||
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
|
||||
${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
|
||||
</textarea>
|
||||
|
||||
@@ -216,7 +213,7 @@ ParseAll=${ztest:parseall:0:1}
|
||||
HTMLFirst=${ztest:htmlfirst:0:1}
|
||||
Cache=${ztest:cache:0:1}
|
||||
NoRecatch=${ztest:norecatch:0:1}
|
||||
Dos=${dos}
|
||||
Dos=${dos
|
||||
Index=${ztest:index:0:1}
|
||||
WordIndex=${ztest:index2:0:1}
|
||||
Log=${ztest:logf:0:1:2}
|
||||
@@ -244,8 +241,6 @@ StripQuery=${stripquery}
|
||||
StoreAllInCache=${ztest:cache2:0:1}
|
||||
Warc=${ztest:warc:0:1}
|
||||
WarcFile=${warcfile}
|
||||
SingleFile=${ztest:singlefile:0:1}
|
||||
SingleFileMaxSize=${singlefilemax}
|
||||
LogType=${logtype}
|
||||
UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
|
||||
ProxyType=${proxytype}
|
||||
|
||||
8
lang.def
8
lang.def
@@ -1042,11 +1042,3 @@ LANG_WARCFILE
|
||||
WARC archive name:
|
||||
LANG_WARCFILETIP
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
LANG_SINGLEFILE
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
LANG_SINGLEFILETIP
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
LANG_SINGLEFILEMAX
|
||||
Largest inlined asset (bytes):
|
||||
LANG_SINGLEFILEMAXTIP
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -1012,11 +1012,3 @@ WARC archive name:
|
||||
WARC archive name:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Largest inlined asset (bytes):
|
||||
Largest inlined asset (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -1012,11 +1012,3 @@ WARC archive name:
|
||||
Nom de l'archive WARC :
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nom de base optionnel pour l'archive WARC ; laissez vide pour le générer automatiquement dans le répertoire de sortie.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Intégrer les ressources en URI data: (pages autonomes)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Une fois le miroir terminé, réécrire chaque page enregistrée avec ses feuilles de style, scripts, images et polices intégrés, afin qu'une page puisse aussi être ouverte seule.
|
||||
Largest inlined asset (bytes):
|
||||
Taille maximale d'une ressource intégrée (octets) :
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Au-delà de cette taille, la ressource reste un lien ordinaire ; laissez vide pour la valeur par défaut de 10485760 octets.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "26 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "23 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -40,7 +40,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-%Z, \-\-single\-file\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
[ \fB\-KN, \-\-keep\-links[=N]\fR ]
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
@@ -199,8 +198,6 @@ delayed type check, don't make any link test but wait for files download to star
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%Z
|
||||
after the mirror, rewrite each saved page with its stylesheets, scripts, images and fonts inlined as data: URIs, so any page also opens on its own (links between pages stay relative; audio and video stay links); \-\-single\-file\-max\-size N caps each asset (default 10485760 bytes) (\-\-single\-file)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
|
||||
@@ -32,9 +32,7 @@ AM_LDFLAGS = \
|
||||
|
||||
bin_PROGRAMS = proxytrack httrack htsserver
|
||||
|
||||
httrack_SOURCES = httrack.c htsbacktrace.c htsbacktrace.h
|
||||
# $(DL_LIBS): dladdr() in the crash handler, still in libdl on pre-2.34 glibc.
|
||||
httrack_LDADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
|
||||
httrack_LDADD = $(THREADS_LIBS) libhttrack.la
|
||||
htsserver_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) libhttrack.la
|
||||
proxytrack_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS)
|
||||
|
||||
@@ -49,7 +47,6 @@ htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
|
||||
lib_LTLIBRARIES = libhttrack.la
|
||||
|
||||
htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h \
|
||||
htscmdline.c htscmdline.h \
|
||||
htsurlport.c htsurlport.h
|
||||
proxytrack_SOURCES = proxy/main.c \
|
||||
proxy/proxytrack.c proxy/store.c \
|
||||
@@ -63,21 +60,21 @@ whttrackrun_SCRIPTS = webhttrack
|
||||
libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htscache_selftest.c htsdns_selftest.c htsselftest.c \
|
||||
htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
|
||||
htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htswarc.c htssinglefile.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmd5.c htscodec.c htswarc.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
|
||||
htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h \
|
||||
htscmdline.h htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
|
||||
htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
|
||||
htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
|
||||
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htssinglefile.h htsproxy.h htszlib.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htsproxy.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
htscharset.h punycode.h htsencoding.h \
|
||||
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
|
||||
|
||||
@@ -123,10 +123,6 @@ const char *hts_optalias[][4] = {
|
||||
{"warc-cdxj", "-%rc", "single", ""},
|
||||
{"wacz", "-%rz", "single",
|
||||
"package the WARC archive, CDXJ index and pages as a WACZ file"},
|
||||
{"single-file", "-%Z", "single",
|
||||
"after the mirror, inline each page's assets as data: URIs"},
|
||||
{"single-file-max-size", "-%Zs", "param1",
|
||||
"per-asset cap for --single-file, in bytes (implies it; default 10485760)"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
|
||||
@@ -541,15 +541,9 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
|
||||
const char *ext) {
|
||||
// do not use tempnam() but a regular filename
|
||||
back->tmpfile_buffer[0] = '\0';
|
||||
if (back->url_sav[0] != '\0') {
|
||||
/* same capacity as url_sav, so truncation drops the extension and aliases
|
||||
the temp name onto the live file that back_finalize_backup() UNLINKs */
|
||||
if (!sprintfbuff(back->tmpfile_buffer, "%s.%s", back->url_sav, ext)) {
|
||||
hts_log_print(opt, LOG_WARNING, "temporary filename too long for %s",
|
||||
back->url_sav);
|
||||
back->tmpfile_buffer[0] = '\0';
|
||||
return -1;
|
||||
}
|
||||
if (back->url_sav != NULL && back->url_sav[0] != '\0') {
|
||||
snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
|
||||
back->url_sav, ext);
|
||||
back->tmpfile = back->tmpfile_buffer;
|
||||
if (structcheck(back->tmpfile) != 0) {
|
||||
hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
|
||||
@@ -557,15 +551,8 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
/* truncation here would collide distinct tmpnameid's onto one name */
|
||||
if (!sprintfbuff(back->tmpfile_buffer, "%s/tmp%d.%s",
|
||||
StringBuff(opt->path_html_utf8), opt->state.tmpnameid++,
|
||||
ext)) {
|
||||
hts_log_print(opt, LOG_WARNING, "temporary filename too long in %s",
|
||||
StringBuff(opt->path_html_utf8));
|
||||
back->tmpfile_buffer[0] = '\0';
|
||||
return -1;
|
||||
}
|
||||
snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
|
||||
StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
|
||||
back->tmpfile = back->tmpfile_buffer;
|
||||
}
|
||||
/* OK */
|
||||
@@ -900,7 +887,8 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
HTS_STAT.stat_bytes += back[p].r.size;
|
||||
HTS_STAT.stat_files++;
|
||||
hts_log_print(opt, LOG_TRACE, "added file %s%s => %s",
|
||||
back[p].url_adr, back[p].url_fil, back[p].url_sav);
|
||||
back[p].url_adr, back[p].url_fil,
|
||||
back[p].url_sav != NULL ? back[p].url_sav : "");
|
||||
}
|
||||
if ((!back[p].r.notmodified) && (opt->is_update)) {
|
||||
HTS_STAT.stat_updated_files++; // page modifiée
|
||||
@@ -2314,24 +2302,26 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
|
||||
&& slot_can_be_finalized(opt, &back[i]);
|
||||
int may_serialize = slot_can_be_cached_on_disk(&back[i]);
|
||||
|
||||
hts_log_print(
|
||||
opt, LOG_DEBUG,
|
||||
"back[%03d]: may_clean=%d, may_finalize_disk=%d, "
|
||||
"may_serialize=%d:" LF "\t"
|
||||
"finalized(%d), status(%d), locked(%d), delayed(%d), "
|
||||
"test(%d), " LF "\t"
|
||||
"statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), " LF
|
||||
"\t"
|
||||
"contenttype(%s), url(%s%s), save(%s)",
|
||||
i, may_clean, may_finalize, may_serialize, back[i].finalized,
|
||||
back[i].status, back[i].locked, IS_DELAYED_EXT(back[i].url_sav),
|
||||
back[i].testmode, back[i].r.statuscode, (int) back[i].r.size,
|
||||
back[i].r.is_write,
|
||||
may_be_hypertext_mime(opt, back[i].r.contenttype,
|
||||
back[i].url_fil),
|
||||
/* */
|
||||
back[i].r.contenttype, back[i].url_adr, back[i].url_fil,
|
||||
back[i].url_sav);
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"back[%03d]: may_clean=%d, may_finalize_disk=%d, may_serialize=%d:"
|
||||
LF "\t"
|
||||
"finalized(%d), status(%d), locked(%d), delayed(%d), test(%d), "
|
||||
LF "\t"
|
||||
"statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), "
|
||||
LF "\t" "contenttype(%s), url(%s%s), save(%s)", i,
|
||||
may_clean, may_finalize, may_serialize,
|
||||
back[i].finalized, back[i].status, back[i].locked,
|
||||
IS_DELAYED_EXT(back[i].url_sav), back[i].testmode,
|
||||
back[i].r.statuscode, (int) back[i].r.size,
|
||||
back[i].r.is_write, may_be_hypertext_mime(opt,
|
||||
back[i].r.
|
||||
contenttype,
|
||||
back[i].
|
||||
url_fil),
|
||||
/* */
|
||||
back[i].r.contenttype, back[i].url_adr,
|
||||
back[i].url_fil,
|
||||
back[i].url_sav ? back[i].url_sav : "<null>");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2867,8 +2857,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// new session
|
||||
back[i].r.ssl_con = SSL_new(openssl_ctx);
|
||||
if (back[i].r.ssl_con) {
|
||||
/* non-const twin: the OpenSSL macro casts the qualifier away */
|
||||
char *hostname = jump_protocol(back[i].url_adr);
|
||||
const char* hostname = jump_protocol_const(back[i].url_adr);
|
||||
// some servers expect the hostname on the clienthello (SNI TLS extension)
|
||||
SSL_set_tlsext_host_name(back[i].r.ssl_con, hostname);
|
||||
SSL_clear(back[i].r.ssl_con);
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: crash backtrace printer */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
/* Before every header: glibc gates dladdr() on it. */
|
||||
#if defined(__linux) && !defined(_GNU_SOURCE)
|
||||
#define _GNU_SOURCE
|
||||
#endif
|
||||
|
||||
#include "htsbacktrace.h"
|
||||
|
||||
#include "htsglobal.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#ifdef _WIN32
|
||||
#include <io.h> /* write */
|
||||
#endif
|
||||
#ifdef HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#if (defined(__linux) && defined(HAVE_EXECINFO_H))
|
||||
#include <dlfcn.h>
|
||||
#include <errno.h>
|
||||
#include <execinfo.h>
|
||||
#include <signal.h>
|
||||
#include <time.h>
|
||||
#include <sys/wait.h>
|
||||
#define USES_BACKTRACE
|
||||
#endif
|
||||
|
||||
#ifdef USES_BACKTRACE
|
||||
#define BT_MAX_FRAMES 64 /* frames we try to name */
|
||||
#define BT_MAX_MODULES 8 /* distinct modules, one child each */
|
||||
#define BT_HEX_SIZE 19 /* "0x" + 16 nibbles + NUL */
|
||||
#define BT_PATH_SIZE 1024 /* module path; longer is skipped */
|
||||
#define BT_WAIT_TICKS 300 /* 10ms ticks, shared: cap a slow child */
|
||||
#define BT_NO_SYMBOLIZER 127 /* child exit: execvp() found none */
|
||||
|
||||
static hts_boolean symbolize_crash = HTS_TRUE;
|
||||
|
||||
/* "0x"-prefixed hex: the handler must stay stdio-free. */
|
||||
static void print_hex(char *buffer, uintptr_t value) {
|
||||
static const char digits[] = "0123456789abcdef";
|
||||
size_t i = 2, a, b;
|
||||
|
||||
buffer[0] = '0';
|
||||
buffer[1] = 'x';
|
||||
do {
|
||||
buffer[i++] = digits[value & 0xf];
|
||||
value >>= 4;
|
||||
} while (value != 0);
|
||||
buffer[i] = '\0';
|
||||
for (a = 2, b = i - 1; a < b; a++, b--) {
|
||||
const char c = buffer[a];
|
||||
|
||||
buffer[a] = buffer[b];
|
||||
buffer[b] = c;
|
||||
}
|
||||
}
|
||||
|
||||
/* HTS_FALSE if src does not fit: a truncated module path would point the
|
||||
symbolizer at the wrong file. */
|
||||
static hts_boolean copy_bounded(char *dest, size_t size, const char *src) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < size - 1 && src[i] != '\0'; i++) {
|
||||
dest[i] = src[i];
|
||||
}
|
||||
dest[i] = '\0';
|
||||
return src[i] == '\0' ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Run the symbolizer on argv, output on fd, within *budget ticks. HTS_FALSE
|
||||
only if none could be run at all; otherwise silent, the raw trace stands. */
|
||||
static hts_boolean spawn_symbolizer(char **argv, int fd, int *budget) {
|
||||
const pid_t pid = fork();
|
||||
int status = 0;
|
||||
|
||||
if (pid == -1)
|
||||
return HTS_FALSE;
|
||||
if (pid == 0) {
|
||||
static char llvm_prog[] = "llvm-symbolizer";
|
||||
static char llvm_opts[] = "-p";
|
||||
|
||||
dup2(fd, 1); /* both symbolizers write on stdout */
|
||||
execvp(argv[0], argv);
|
||||
argv[0] = llvm_prog; /* an LLVM-only install ships no addr2line */
|
||||
argv[1] = llvm_opts;
|
||||
execvp(argv[0], argv);
|
||||
_exit(BT_NO_SYMBOLIZER);
|
||||
}
|
||||
for (; *budget > 0; (*budget)--) {
|
||||
const struct timespec tick = {0, 10 * 1000 * 1000};
|
||||
const pid_t reaped = waitpid(pid, &status, WNOHANG);
|
||||
|
||||
if (reaped == pid)
|
||||
return WIFEXITED(status) && WEXITSTATUS(status) == BT_NO_SYMBOLIZER
|
||||
? HTS_FALSE
|
||||
: HTS_TRUE;
|
||||
if (reaped == -1 && errno != EINTR)
|
||||
return HTS_TRUE;
|
||||
nanosleep(&tick, NULL);
|
||||
}
|
||||
kill(pid, SIGKILL);
|
||||
waitpid(pid, NULL, 0);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Name the frames backtrace_symbols_fd() leaves as module+offset:
|
||||
-fvisibility=hidden keeps them out of .dynsym, but DWARF has them. dladdr()
|
||||
is not formally async-signal-safe; accepted, this path is already fatal. */
|
||||
static void symbolize_backtrace(void *const *stack, int size, int fd) {
|
||||
static char prog[] = "addr2line";
|
||||
static char opts[] = "-Cfipa";
|
||||
static char dashe[] = "-e";
|
||||
char hex[BT_MAX_FRAMES][BT_HEX_SIZE];
|
||||
const void *base[BT_MAX_FRAMES];
|
||||
const char *name[BT_MAX_FRAMES];
|
||||
hts_boolean grouped[BT_MAX_FRAMES];
|
||||
char module[BT_PATH_SIZE];
|
||||
char *argv[4 + BT_MAX_FRAMES + 1];
|
||||
int budget = BT_WAIT_TICKS;
|
||||
int i, spawned;
|
||||
|
||||
if (size > BT_MAX_FRAMES)
|
||||
size = BT_MAX_FRAMES;
|
||||
|
||||
for (i = 0; i < size; i++) {
|
||||
Dl_info info;
|
||||
|
||||
grouped[i] = HTS_TRUE; /* skipped unless dladdr() places the frame */
|
||||
if (dladdr(stack[i], &info) == 0 || info.dli_fname == NULL ||
|
||||
info.dli_fname[0] == '\0')
|
||||
continue;
|
||||
base[i] = info.dli_fbase;
|
||||
name[i] = info.dli_fname;
|
||||
print_hex(hex[i], (uintptr_t) ((const char *) stack[i] -
|
||||
(const char *) info.dli_fbase));
|
||||
grouped[i] = HTS_FALSE;
|
||||
}
|
||||
|
||||
/* One child per module: addr2line takes a single -e. Each frame is claimed
|
||||
once, so argc cannot exceed argv[]. */
|
||||
for (spawned = 0; spawned < BT_MAX_MODULES; spawned++) {
|
||||
int first, j, argc = 0;
|
||||
|
||||
for (first = 0; first < size && grouped[first]; first++)
|
||||
;
|
||||
if (first >= size)
|
||||
break;
|
||||
argv[argc++] = prog;
|
||||
argv[argc++] = opts;
|
||||
argv[argc++] = dashe;
|
||||
argv[argc++] = module;
|
||||
for (j = first; j < size; j++) {
|
||||
if (grouped[j] || base[j] != base[first])
|
||||
continue;
|
||||
grouped[j] = HTS_TRUE;
|
||||
argv[argc++] = hex[j];
|
||||
}
|
||||
argv[argc] = NULL;
|
||||
/* access(): skip pseudo-modules like linux-vdso, which have no file and
|
||||
would draw nothing but an addr2line complaint. */
|
||||
if (copy_bounded(module, sizeof(module), name[first]) &&
|
||||
access(module, R_OK) == 0) {
|
||||
const size_t len = strlen(module);
|
||||
|
||||
/* addr2line -a prints offsets only: say which module they are in. */
|
||||
(void) (write(fd, module, len) == (ssize_t) len);
|
||||
(void) (write(fd, ":\n", 2) == 2);
|
||||
if (!spawn_symbolizer(argv, fd, &budget))
|
||||
break; /* no symbolizer: stop at one header */
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void hts_backtrace_init(void) {
|
||||
#ifdef USES_BACKTRACE
|
||||
symbolize_crash =
|
||||
getenv("HTTRACK_NO_SYMBOLIZE") == NULL ? HTS_TRUE : HTS_FALSE;
|
||||
#endif
|
||||
}
|
||||
|
||||
void hts_print_backtrace(int fd) {
|
||||
#ifdef USES_BACKTRACE
|
||||
void *stack[256];
|
||||
const int size = backtrace(stack, sizeof(stack) / sizeof(stack[0]));
|
||||
|
||||
/* A fault inside the handler lands back here: symbolizing twice interleaves
|
||||
two traces on fd and spends a second budget. */
|
||||
static volatile sig_atomic_t entered = 0;
|
||||
|
||||
if (size != 0) {
|
||||
backtrace_symbols_fd(stack, size, fd);
|
||||
if (symbolize_crash && entered == 0) {
|
||||
entered = 1;
|
||||
symbolize_backtrace(stack, size, fd);
|
||||
entered = 0;
|
||||
}
|
||||
}
|
||||
#else
|
||||
const char msg[] = "No stack trace available on this OS :(\n";
|
||||
|
||||
if (write(fd, msg, sizeof(msg) - 1) != sizeof(msg) - 1) {
|
||||
/* sorry GCC */
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: crash backtrace printer */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSBACKTRACE_DEFH
|
||||
#define HTSBACKTRACE_DEFH
|
||||
|
||||
/* Sample HTTRACK_NO_SYMBOLIZE before any crash: getenv() is not signal-safe.
|
||||
Call once, from the process that installs the fatal-signal handlers. */
|
||||
void hts_backtrace_init(void);
|
||||
|
||||
/* Write the calling thread's stack to fd, callable from a fatal signal handler:
|
||||
raw frames first, then whatever an external symbolizer can name. Allocates
|
||||
nothing; prints a one-line notice where the OS has no backtrace(). */
|
||||
void hts_print_backtrace(int fd);
|
||||
|
||||
#endif
|
||||
@@ -629,8 +629,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
// File exists on disk with declared cache name (this is expected!)
|
||||
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) { // un fichier existe déja
|
||||
// Expected size ?
|
||||
const LLint fsize = fsize_utf8(
|
||||
fconv(catbuff, sizeof(catbuff), previous_save));
|
||||
const size_t fsize =
|
||||
fsize_utf8(fconv(catbuff, sizeof(catbuff), previous_save));
|
||||
if (fsize == r.size) {
|
||||
// Target name is the previous name, and the file looks good: nothing to do!
|
||||
if (strcmp(previous_save, target_save) == 0) {
|
||||
@@ -666,8 +666,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
// Suppose a broken mirror, with a file being renamed: OK
|
||||
else if (fexist_utf8(fconv(catbuff, sizeof(catbuff), target_save))) {
|
||||
// Expected size ?
|
||||
const LLint fsize =
|
||||
fsize_utf8(fconv(catbuff, sizeof(catbuff), target_save));
|
||||
const size_t fsize = fsize_utf8(fconv(catbuff, sizeof(catbuff), target_save));
|
||||
|
||||
if (fsize == r.size) {
|
||||
// So far so good
|
||||
@@ -1441,7 +1440,7 @@ int cache_brstr(char *adr, char *s, size_t s_size) {
|
||||
|
||||
/* binput bounded to a NUL-terminated buffer: refuse to start a read at or
|
||||
past `end`, so a prior over-advance can't walk a cache-index parse OOB. */
|
||||
int cache_binput(const char *adr, const char *end, char *s, int max) {
|
||||
int cache_binput(char *adr, const char *end, char *s, int max) {
|
||||
if (adr >= end) {
|
||||
s[0] = '\0';
|
||||
return 0;
|
||||
|
||||
@@ -93,7 +93,7 @@ void cache_rstr(FILE *fp, char *s, size_t s_size);
|
||||
char *cache_rstr_addr(FILE * fp);
|
||||
int cache_brstr(char *adr, char *s, size_t s_size);
|
||||
/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
|
||||
int cache_binput(const char *adr, const char *end, char *s, int max);
|
||||
int cache_binput(char *adr, const char *end, char *s, int max);
|
||||
int cache_brint(char *adr, int *i);
|
||||
void cache_rint(FILE * fp, int *i);
|
||||
void cache_rLLint(FILE * fp, LLint * i);
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: command line splitter, shared by the engine and */
|
||||
/* htsserver */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#include "htscmdline.h"
|
||||
|
||||
#include "htssafe.h"
|
||||
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
|
||||
char **hts_split_cmdline(char *cmd, int *nargs) {
|
||||
size_t nsep = 0;
|
||||
size_t capacity;
|
||||
size_t r;
|
||||
size_t w;
|
||||
int argc = 0;
|
||||
hts_boolean quoted = HTS_FALSE;
|
||||
char **argv;
|
||||
|
||||
*nargs = 0;
|
||||
|
||||
/* fold the other separators, so counting them sizes the vector exactly */
|
||||
for (r = 0; cmd[r] != '\0'; r++) {
|
||||
if (cmd[r] == '\t' || cmd[r] == '\r' || cmd[r] == '\n') {
|
||||
cmd[r] = ' ';
|
||||
}
|
||||
if (cmd[r] == ' ') {
|
||||
nsep++;
|
||||
}
|
||||
}
|
||||
|
||||
/* at most one argument per separator, plus the leading one and the NULL */
|
||||
if (nsep > (size_t) INT_MAX - 1 || nsep > SIZE_MAX / sizeof(char *) - 2) {
|
||||
return NULL;
|
||||
}
|
||||
capacity = nsep + 2;
|
||||
argv = (char **) malloct(capacity * sizeof(char *));
|
||||
if (argv == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
argv[argc++] = cmd;
|
||||
for (r = 0, w = 0; cmd[r] != '\0';) {
|
||||
if (quoted && cmd[r] == '\\' &&
|
||||
(cmd[r + 1] == '\\' || cmd[r + 1] == '\"')) {
|
||||
r++;
|
||||
cmd[w++] = cmd[r++];
|
||||
} else if (cmd[r] == '\"') {
|
||||
quoted = !quoted;
|
||||
cmd[w++] = cmd[r++];
|
||||
} else if (cmd[r] == ' ' && !quoted) {
|
||||
cmd[w++] = '\0';
|
||||
assertf((size_t) argc < capacity - 1); /* the last slot holds the NULL */
|
||||
argv[argc++] = cmd + w;
|
||||
r++;
|
||||
} else {
|
||||
cmd[w++] = cmd[r++];
|
||||
}
|
||||
}
|
||||
cmd[w] = '\0';
|
||||
argv[argc] = NULL; /* callers may rely on argv[argc] == NULL */
|
||||
|
||||
*nargs = argc;
|
||||
return argv;
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: command line splitter, shared by the engine and */
|
||||
/* htsserver */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSCMDLINE_DEFH
|
||||
#define HTSCMDLINE_DEFH
|
||||
|
||||
#include "htsglobal.h"
|
||||
|
||||
/* Split "cmd" in place into a NULL-terminated argv vector of *nargs entries,
|
||||
argv[0] being the program name and quotes left for the engine to strip.
|
||||
Returns a malloct'ed vector of pointers into cmd (freet the vector, never its
|
||||
entries), or NULL when it cannot be sized or allocated. */
|
||||
char **hts_split_cmdline(char *cmd, int *nargs);
|
||||
|
||||
#endif
|
||||
@@ -40,7 +40,6 @@ Please visit our Website: http://www.httrack.com
|
||||
/* File defs */
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -448,22 +447,13 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
|
||||
const char *fil) {
|
||||
if (!*makeindex_done) {
|
||||
if (*makeindex_fp) {
|
||||
/* sized off link_escaped below: at the old flat 1024 a long first link
|
||||
produced a redirect to a clipped URL */
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2 + 64];
|
||||
char BIGSTK tempo[1024];
|
||||
if (makeindex_links == 1) {
|
||||
char BIGSTK link_escaped[HTS_URLMAXSIZE * 2];
|
||||
escape_uri_utf(makeindex_firstlink, link_escaped, sizeof(link_escaped));
|
||||
/* no redirect beats one pointing at a clipped URL */
|
||||
if (!sprintfbuff(
|
||||
tempo,
|
||||
"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">" CRLF,
|
||||
link_escaped)) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"index redirect omitted: first link too long (%s)",
|
||||
makeindex_firstlink);
|
||||
tempo[0] = '\0';
|
||||
}
|
||||
snprintf(tempo, sizeof(tempo),
|
||||
"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">" CRLF,
|
||||
link_escaped);
|
||||
} else
|
||||
tempo[0] = '\0';
|
||||
hts_template_format(*makeindex_fp, template_footer,
|
||||
@@ -706,19 +696,17 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// copier adresse(s) dans liste des adresses
|
||||
{
|
||||
char *a = url1;
|
||||
const LLint list_sz = StringNotEmpty(opt->filelist)
|
||||
? fsize_utf8(StringBuff(opt->filelist))
|
||||
: 0;
|
||||
/* two bytes reserved per list byte; -1 makes an undoublable size refused */
|
||||
const LLint list_room =
|
||||
list_sz > 0 ? (list_sz <= INT64_MAX / 2 ? list_sz * 2 : -1) : 0;
|
||||
const size_t primary_len =
|
||||
llint_grow_size_t(8192 + strlen(url1) * 2, list_room, 0);
|
||||
int primary_len = 8192;
|
||||
|
||||
if (StringNotEmpty(opt->filelist)) {
|
||||
primary_len += max(0, fsize_utf8(StringBuff(opt->filelist)) * 2);
|
||||
}
|
||||
primary_len += (int) strlen(url1) * 2;
|
||||
|
||||
// création de la première page, qui contient les liens de base à scanner
|
||||
// c'est plus propre et plus logique que d'entrer à la main les liens dans la pile
|
||||
// on bénéficie ainsi des vérifications et des tests du robot pour les liens "primaires"
|
||||
primary = primary_len != (size_t) -1 ? (char *) malloct(primary_len) : NULL;
|
||||
primary = (char *) malloct(primary_len);
|
||||
if (!primary) {
|
||||
printf("PANIC! : Not enough memory [%d]\n", __LINE__);
|
||||
XH_extuninit;
|
||||
@@ -899,7 +887,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
|
||||
if (filelist_buff != NULL) {
|
||||
size_t filelist_ptr = 0;
|
||||
int filelist_ptr = 0;
|
||||
int n = 0;
|
||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
||||
|
||||
@@ -2183,10 +2171,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
// fin purge!
|
||||
|
||||
/* --single-file: inline each page's assets now that the tree is final (after
|
||||
the purge, so we never rewrite a file that is about to be deleted). */
|
||||
singlefile_process_mirror(opt);
|
||||
|
||||
// Indexation
|
||||
if (opt->kindex)
|
||||
index_finish(StringBuff(opt->path_html), opt->kindex);
|
||||
@@ -3648,10 +3632,6 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
to->warc_cdx = from->warc_cdx;
|
||||
to->warc_wacz = from->warc_wacz;
|
||||
|
||||
to->single_file = from->single_file;
|
||||
if (from->single_file_max_size > 0)
|
||||
to->single_file_max_size = from->single_file_max_size;
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
to->pause_max_ms = from->pause_max_ms;
|
||||
|
||||
@@ -145,7 +145,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
int argv_url = -1; // ==0 : utiliser cache et doit.log
|
||||
char *argv_firsturl = NULL; // utilisé pour nommage par défaut
|
||||
char *url = NULL; // URLS séparées par un espace
|
||||
size_t url_sz = 65535;
|
||||
int url_sz = 65535;
|
||||
|
||||
// the parametres
|
||||
int httrack_logmode = 3; // ONE log file
|
||||
@@ -224,22 +224,21 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|
||||
/* create x_argvblk buffer for transformed command line */
|
||||
{
|
||||
size_t current_size = 0;
|
||||
const LLint size = fsize("config");
|
||||
size_t blk_size;
|
||||
int current_size = 0;
|
||||
int size;
|
||||
int na;
|
||||
|
||||
for(na = 0; na < argc; na++)
|
||||
current_size += strlen(argv[na]) + 1;
|
||||
/* a huge file named "config" must saturate, not wrap, the capacity */
|
||||
blk_size = llint_grow_size_t(current_size, size > 0 ? size : 0, 32768);
|
||||
x_argvblk = blk_size != (size_t) -1 ? (char *) malloct(blk_size) : NULL;
|
||||
current_size += (int) (strlen(argv[na]) + 1);
|
||||
if ((size = fsize("config")) > 0)
|
||||
current_size += size;
|
||||
x_argvblk = (char *) malloct(current_size + 32768);
|
||||
if (x_argvblk == NULL) {
|
||||
HTS_PANIC_PRINTF("Error, not enough memory");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
x_argvblk_size = blk_size;
|
||||
x_argvblk_size = (size_t) (current_size + 32768);
|
||||
x_argvblk[0] = '\0';
|
||||
x_ptr = 0;
|
||||
|
||||
@@ -1457,29 +1456,20 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
FILE *fp = FOPEN(argv[na], "rb");
|
||||
|
||||
if (fp != NULL) {
|
||||
size_t cl = strlen(url);
|
||||
const size_t fzs = llint_to_size_t(fz);
|
||||
const size_t capa = llint_grow_size_t(cl, fz, 8192);
|
||||
int cl = (int) strlen(url);
|
||||
|
||||
if (capa == (size_t) -1) {
|
||||
fclose(fp);
|
||||
HTS_PANIC_PRINTF("File url list too large");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
ensureUrlCapacity(url, url_sz, capa);
|
||||
ensureUrlCapacity(url, url_sz, cl + fz + 8192);
|
||||
if (cl > 0) { /* don't stick! (3.43) */
|
||||
url[cl] = ' ';
|
||||
cl++;
|
||||
}
|
||||
if (fread(url + cl, 1, fzs, fp) != fzs) {
|
||||
fclose(fp);
|
||||
if (fread(url + cl, 1, fz, fp) != fz) {
|
||||
HTS_PANIC_PRINTF("File url list could not be read");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
fclose(fp);
|
||||
*(url + cl + fzs) = '\0';
|
||||
*(url + cl + fz) = '\0';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1795,36 +1785,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
break;
|
||||
case 'Z': // single-file: inline each page's assets as data: URIs
|
||||
if (*(com + 1) == 's') { // --single-file-max-size N: per-asset
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option single-file-max-size needs a blank "
|
||||
"space and a size");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
{ // reject non-numeric/negative/overflow; keep the default
|
||||
char *end;
|
||||
LLint v;
|
||||
|
||||
errno = 0;
|
||||
v = strtoll(argv[na], &end, 10);
|
||||
if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
|
||||
errno != ERANGE && v > 0)
|
||||
opt->single_file_max_size = v;
|
||||
}
|
||||
opt->single_file = HTS_TRUE;
|
||||
} else {
|
||||
opt->single_file = HTS_TRUE;
|
||||
if (*(com + 1) == '0') {
|
||||
opt->single_file = HTS_FALSE;
|
||||
com++;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
@@ -2336,8 +2296,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|
||||
} else { // URL/filters
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const size_t urlSize = strlen(argv[na]);
|
||||
const size_t capa = strlen(url) + urlSize + 32;
|
||||
const int urlSize = (int) strlen(argv[na]);
|
||||
const int capa = (int) (strlen(url) + urlSize + 32);
|
||||
|
||||
assertf(urlSize < HTS_URLMAXSIZE);
|
||||
if (urlSize < HTS_URLMAXSIZE) {
|
||||
|
||||
@@ -72,8 +72,7 @@ Please visit our Website: http://www.httrack.com
|
||||
HTS_UNUSED: suppress unused-symbol warnings. HTS_STATIC: an unused-safe
|
||||
static. HTS_PRINTF_FUN(fmt, arg): mark a printf-like function so the
|
||||
compiler type-checks the format string at argument index fmt against the
|
||||
varargs starting at arg. HTS_CHECK_RESULT: the return value carries the only
|
||||
error signal, so dropping it is a bug; a (void) cast does not silence it. */
|
||||
varargs starting at arg. */
|
||||
#ifndef HTS_UNUSED
|
||||
#ifdef __GNUC__
|
||||
#define HTS_UNUSED __attribute__((unused))
|
||||
@@ -81,13 +80,10 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_STATIC static __attribute__((unused))
|
||||
|
||||
#define HTS_PRINTF_FUN(fmt, arg) __attribute__((format(printf, fmt, arg)))
|
||||
|
||||
#define HTS_CHECK_RESULT __attribute__((warn_unused_result))
|
||||
#else
|
||||
#define HTS_UNUSED
|
||||
#define HTS_STATIC static
|
||||
#define HTS_PRINTF_FUN(fmt, arg)
|
||||
#define HTS_CHECK_RESULT
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@@ -340,20 +336,16 @@ typedef int INTsys;
|
||||
#endif
|
||||
|
||||
/* Socket-handle type. An unsigned integer wide enough for a Windows SOCKET;
|
||||
a plain int file descriptor on POSIX. T_SOCP is its printf conversion,
|
||||
'%' included: unsigned __int64 on Win64 must not be printed with "%d". */
|
||||
a plain int file descriptor on POSIX. */
|
||||
#ifdef _WIN32
|
||||
#if defined(_WIN64)
|
||||
|
||||
typedef unsigned __int64 T_SOC;
|
||||
#define T_SOCP "%" PRIu64
|
||||
#else
|
||||
typedef unsigned __int32 T_SOC;
|
||||
#define T_SOCP "%" PRIu32
|
||||
#endif
|
||||
#else
|
||||
typedef int T_SOC;
|
||||
#define T_SOCP "%d"
|
||||
#endif
|
||||
|
||||
/* Buffer size for a printed network address (IPv4 or IPv6, NUL included). */
|
||||
|
||||
@@ -124,9 +124,7 @@ typedef struct help_wizard_buffers {
|
||||
char stropt[2048]; // options
|
||||
char stropt2[2048]; // options longues
|
||||
char strwild[2048]; // wildcards
|
||||
/* holds all four of the above plus separators: at 4096 a long answer set
|
||||
clipped the filters off the command line */
|
||||
char cmd[HTS_URLMAXSIZE * 2 + 3 * 2048 + 4];
|
||||
char cmd[4096];
|
||||
char str[256];
|
||||
char *argv[256];
|
||||
} help_wizard_buffers;
|
||||
@@ -158,7 +156,9 @@ void help_wizard(httrackp * opt) {
|
||||
char *a;
|
||||
|
||||
//
|
||||
if (buffers == NULL) {
|
||||
if (urls == NULL || mainpath == NULL || projname == NULL || stropt == NULL
|
||||
|| stropt2 == NULL || strwild == NULL || cmd == NULL || str == NULL
|
||||
|| argv == NULL) {
|
||||
fprintf(stderr, "* memory exhausted in %s, line %d\n", __FILE__, __LINE__);
|
||||
return;
|
||||
}
|
||||
@@ -251,7 +251,6 @@ void help_wizard(httrackp * opt) {
|
||||
strcatbuff(stropt2, "--update ");
|
||||
break;
|
||||
case 0:
|
||||
freet(buffers);
|
||||
return;
|
||||
break;
|
||||
}
|
||||
@@ -310,23 +309,14 @@ void help_wizard(httrackp * opt) {
|
||||
printf("\n");
|
||||
if (strlen(stropt) == 1)
|
||||
stropt[0] = '\0'; // aucune
|
||||
/* the tail is the filter list, and cmd is split into the argv handed to
|
||||
hts_main() below: a clipped line would silently widen the crawl */
|
||||
if (!sprintfbuff(cmd, "%s %s %s %s", urls, stropt, stropt2, strwild)) {
|
||||
printf("* command line too long (%d bytes max)\n",
|
||||
(int) sizeof(cmd) - 1);
|
||||
freet(buffers);
|
||||
return;
|
||||
}
|
||||
snprintf(cmd, sizeof(cmd), "%s %s %s %s", urls, stropt, stropt2, strwild);
|
||||
printf("---> Wizard command line: httrack %s\n\n", cmd);
|
||||
printf("Ready to launch the mirror? (Y/n) :");
|
||||
fflush(stdout);
|
||||
linput(stdin, str, 250);
|
||||
if (strnotempty(str)) {
|
||||
if (!((str[0] == 'y') || (str[0] == 'Y'))) {
|
||||
freet(buffers);
|
||||
if (!((str[0] == 'y') || (str[0] == 'Y')))
|
||||
return;
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
@@ -350,7 +340,7 @@ void help_wizard(httrackp * opt) {
|
||||
}
|
||||
|
||||
/* Free buffers */
|
||||
freet(buffers);
|
||||
free(buffers);
|
||||
#undef urls
|
||||
#undef mainpath
|
||||
#undef projname
|
||||
@@ -534,11 +524,6 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %Z after the mirror, rewrite each saved page with its "
|
||||
"stylesheets, scripts, images and fonts inlined as data: URIs, so "
|
||||
"any page also opens on its own (links between pages stay relative; "
|
||||
"audio and video stay links); --single-file-max-size N caps each "
|
||||
"asset (default 10485760 bytes)");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
|
||||
42
src/htslib.c
42
src/htslib.c
@@ -37,7 +37,6 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -705,16 +704,18 @@ T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
|
||||
/* Check for errors */
|
||||
if (soc == INVALID_SOCKET) {
|
||||
if (retour) {
|
||||
if (!strnotempty(retour->msg)) {
|
||||
if (retour->msg) {
|
||||
if (!strnotempty(retour->msg)) {
|
||||
#ifdef _WIN32
|
||||
int last_errno = WSAGetLastError();
|
||||
int last_errno = WSAGetLastError();
|
||||
|
||||
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
|
||||
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
|
||||
#else
|
||||
int last_errno = errno;
|
||||
int last_errno = errno;
|
||||
|
||||
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
|
||||
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2206,7 +2207,7 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
#if DEBUG
|
||||
printf("erreur gethostbyname\n");
|
||||
#endif
|
||||
if (retour != NULL) {
|
||||
if (retour && retour->msg) {
|
||||
#ifdef _WIN32
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to get server's address: %s", error);
|
||||
@@ -2237,7 +2238,7 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
DEBUG_W("socket()=%d\n" _(int) soc);
|
||||
#endif
|
||||
if (soc == INVALID_SOCKET) {
|
||||
if (retour != NULL) {
|
||||
if (retour && retour->msg) {
|
||||
#ifdef _WIN32
|
||||
int last_errno = WSAGetLastError();
|
||||
|
||||
@@ -2261,8 +2262,17 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
&bind_addr, &error) == NULL
|
||||
|| bind(soc, &SOCaddr_sockaddr(bind_addr),
|
||||
SOCaddr_size(bind_addr)) != 0) {
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to bind the specificied server address: %s", error);
|
||||
if (retour && retour->msg) {
|
||||
#ifdef _WIN32
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to bind the specificied server address: %s",
|
||||
error);
|
||||
#else
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to bind the specificied server address: %s",
|
||||
error);
|
||||
#endif
|
||||
}
|
||||
deletesoc(soc);
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
@@ -2309,7 +2319,7 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
#if HDEBUG
|
||||
printf("unable to connect!\n");
|
||||
#endif
|
||||
if (retour != NULL) {
|
||||
if (retour != NULL && retour->msg) {
|
||||
#ifdef _WIN32
|
||||
const int last_errno = WSAGetLastError();
|
||||
|
||||
@@ -2588,15 +2598,13 @@ void deletesoc(T_SOC soc) {
|
||||
if (closesocket(soc) != 0) {
|
||||
int err = WSAGetLastError();
|
||||
|
||||
fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
|
||||
strerror(err));
|
||||
fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err));
|
||||
}
|
||||
#else
|
||||
if (close(soc) != 0) {
|
||||
const int err = errno;
|
||||
|
||||
fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
|
||||
strerror(err));
|
||||
fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err));
|
||||
}
|
||||
#endif
|
||||
#if HTS_WIDE_DEBUG
|
||||
@@ -3009,7 +3017,7 @@ int finput(T_SOC fd, char *s, int max) {
|
||||
}
|
||||
|
||||
// Like linput, but in memory (optimized)
|
||||
int binput(const char *buff, char *s, int max) {
|
||||
int binput(char *buff, char *s, int max) {
|
||||
int count = 0;
|
||||
int destCount = 0;
|
||||
|
||||
@@ -6012,8 +6020,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->warc_file, "");
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
opt->single_file = HTS_FALSE;
|
||||
opt->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
|
||||
17
src/htslib.h
17
src/htslib.h
@@ -262,7 +262,7 @@ HTS_INLINE void time_rfc822_local(char *s, struct tm *A);
|
||||
|
||||
HTS_INLINE int sendc(htsblk * r, const char *s);
|
||||
int finput(T_SOC fd, char *s, int max);
|
||||
int binput(const char *buff, char *s, int max);
|
||||
int binput(char *buff, char *s, int max);
|
||||
int linput(FILE * fp, char *s, int max);
|
||||
int linputsoc(T_SOC soc, char *s, int max);
|
||||
int linputsoc_t(T_SOC soc, char *s, int max, int timeout);
|
||||
@@ -607,21 +607,6 @@ static HTS_UNUSED size_t llint_to_size_t(LLint o) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Capacity for @p used bytes plus @p extra more plus @p slack spare;
|
||||
(size_t) -1 if the total exceeds (size_t) -2 or @p extra is negative
|
||||
(llint_to_size_t() would map that to a huge valid-looking size). */
|
||||
static HTS_UNUSED size_t llint_grow_size_t(size_t used, LLint extra,
|
||||
size_t slack) {
|
||||
const size_t max = (size_t) -2; /* (size_t) -1 is the error value */
|
||||
const size_t e = extra >= 0 ? llint_to_size_t(extra) : (size_t) -1;
|
||||
|
||||
if (e == (size_t) -1 || used > max || slack > max - used ||
|
||||
e > max - used - slack) {
|
||||
return (size_t) -1;
|
||||
}
|
||||
return used + e + slack;
|
||||
}
|
||||
|
||||
/* dirent() compatibility */
|
||||
#ifdef _WIN32
|
||||
/* Holds a UTF-8 d_name: MAX_PATH (260) UTF-16 units expand to <=3 bytes each.
|
||||
|
||||
@@ -547,12 +547,6 @@ struct httrackp {
|
||||
archive. Tail: ABI */
|
||||
hts_boolean warc_wacz; /**< --wacz: package archive+index+pages as a WACZ zip
|
||||
(implies --warc + --warc-cdx). Tail: ABI */
|
||||
hts_boolean single_file; /**< --single-file: once the mirror is done, rewrite
|
||||
each saved page with its assets inlined as
|
||||
data: URIs. Tail: ABI */
|
||||
LLint single_file_max_size; /**< --single-file-max-size: per-asset cap in
|
||||
bytes; a bigger asset stays a link.
|
||||
Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
|
||||
@@ -4583,7 +4583,7 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
/* seen as in error */
|
||||
in_error = back[b].r.statuscode;
|
||||
in_error_msg[0] = 0;
|
||||
strncatbuff(in_error_msg, back[b].r.msg, sizeof(in_error_msg) - 1);
|
||||
strncat(in_error_msg, back[b].r.msg, sizeof(in_error_msg) - 1);
|
||||
in_error_size = back[b].r.totalsize;
|
||||
/* don't break, even with "don't take error pages" switch, because we need to process the slot anyway (and cache the error) */
|
||||
}
|
||||
|
||||
@@ -33,7 +33,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#ifndef HTSSAFE_DEFH
|
||||
#define HTSSAFE_DEFH
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -117,15 +116,6 @@ static HTS_UNUSED void abortf_(const char *exp, const char *file, int line) {
|
||||
#endif
|
||||
#define HTS_IS_NOT_CHAR_BUFFER(VAR) (!HTS_IS_CHAR_BUFFER(VAR))
|
||||
|
||||
/* Source capacity for the buff() family, (size_t)-1 when unknown; sizeof of the
|
||||
TYPE keeps a decayed operand ("buf + 1") off -Wsizeof-array-decay. */
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
#define HTS_SIZEOF_SRC_(B) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(__typeof__(B)))
|
||||
#else
|
||||
#define HTS_SIZEOF_SRC_(B) (HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B))
|
||||
#endif
|
||||
|
||||
/* Compile-time checks. */
|
||||
static HTS_UNUSED void htssafe_compile_time_check_(void) {
|
||||
char array[32];
|
||||
@@ -215,17 +205,19 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
|
||||
#define strncatbuff(A, B, N) \
|
||||
__builtin_choose_expr(HTS_IS_CHAR_BUFFER(A), \
|
||||
strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), N, \
|
||||
"overflow while appending '" #B \
|
||||
"' to '" #A "'", \
|
||||
__FILE__, __LINE__), \
|
||||
strncatbuff_ptr_((A), (B), (N)))
|
||||
__builtin_choose_expr( \
|
||||
HTS_IS_CHAR_BUFFER(A), \
|
||||
strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), N, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__), \
|
||||
strncatbuff_ptr_((A), (B), (N)))
|
||||
#else
|
||||
#define strncatbuff(A, B, N) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(A) \
|
||||
? strncat(A, B, N) \
|
||||
: strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), N, \
|
||||
: strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), N, \
|
||||
"overflow while appending '" #B "' to '" #A "'", \
|
||||
__FILE__, __LINE__))
|
||||
#endif
|
||||
@@ -240,7 +232,9 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#define strcatbuff(A, B) \
|
||||
__builtin_choose_expr( \
|
||||
HTS_IS_CHAR_BUFFER(A), \
|
||||
strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), (size_t) -1, \
|
||||
strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
(size_t) -1, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__), \
|
||||
strcatbuff_ptr_((A), (B)))
|
||||
@@ -248,7 +242,9 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#define strcatbuff(A, B) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(A) \
|
||||
? strcat(A, B) \
|
||||
: strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), (size_t) -1, \
|
||||
: strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
(size_t) -1, \
|
||||
"overflow while appending '" #B "' to '" #A "'", \
|
||||
__FILE__, __LINE__))
|
||||
#endif
|
||||
@@ -261,17 +257,19 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
|
||||
#define strcpybuff(A, B) \
|
||||
__builtin_choose_expr(HTS_IS_CHAR_BUFFER(A), \
|
||||
strcpy_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), \
|
||||
"overflow while copying '" #B "' to '" #A \
|
||||
"'", \
|
||||
__FILE__, __LINE__), \
|
||||
strcpybuff_ptr_((A), (B)))
|
||||
__builtin_choose_expr( \
|
||||
HTS_IS_CHAR_BUFFER(A), \
|
||||
strcpy_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
"overflow while copying '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__), \
|
||||
strcpybuff_ptr_((A), (B)))
|
||||
#else
|
||||
#define strcpybuff(A, B) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(A) \
|
||||
? strcpy(A, B) \
|
||||
: strcpy_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), \
|
||||
: strcpy_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
"overflow while copying '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__))
|
||||
#endif
|
||||
@@ -288,24 +286,24 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
* Append characters of "B" to "A", "A" having a maximum capacity of "S".
|
||||
*/
|
||||
#define strlcatbuff(A, B, S) \
|
||||
strncat_safe_(A, S, B, HTS_SIZEOF_SRC_(B), (size_t) -1, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__)
|
||||
strncat_safe_(A, S, B, HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
(size_t) -1, "overflow while appending '" #B "' to '" #A "'", \
|
||||
__FILE__, __LINE__)
|
||||
|
||||
/**
|
||||
* Append at most "N" characters of "B" to "A", "A" having a maximum capacity
|
||||
* of "S".
|
||||
*/
|
||||
#define strlncatbuff(A, B, S, N) \
|
||||
strncat_safe_(A, S, B, HTS_SIZEOF_SRC_(B), N, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
strncat_safe_(A, S, B, HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
N, "overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__)
|
||||
|
||||
/**
|
||||
* Copy characters of "B" to "A", "A" having a maximum capacity of "S".
|
||||
*/
|
||||
#define strlcpybuff(A, B, S) \
|
||||
strcpy_safe_(A, S, B, HTS_SIZEOF_SRC_(B), \
|
||||
strcpy_safe_(A, S, B, HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
"overflow while copying '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__)
|
||||
|
||||
@@ -424,9 +422,7 @@ static HTS_INLINE HTS_UNUSED htsbuff htsbuff_ptr_(char *buf, size_t cap) {
|
||||
*/
|
||||
static HTS_INLINE HTS_UNUSED void htsbuff_catn(htsbuff *b, const char *s,
|
||||
size_t n) {
|
||||
/* the (size_t)-1 "no limit" sentinel would reach strnlen as a bound past
|
||||
PTRDIFF_MAX */
|
||||
const size_t add = n != (size_t) -1 ? strnlen(s, n) : strlen(s);
|
||||
const size_t add = strnlen(s, n);
|
||||
/* Overflow-safe: keep the (potentially huge) 'add' alone on one side. The
|
||||
maintained invariant len < cap makes 'cap - len' >= 1 (no underflow), so
|
||||
'add < cap - len' cannot wrap the way 'len + add < cap' could. */
|
||||
@@ -460,37 +456,6 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
|
||||
return b->buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Formatted print into dest (capacity size, NUL included), truncating to fit
|
||||
* and always NUL-terminating. Returns HTS_TRUE if the whole output fit; the
|
||||
* result is the only truncation signal, so it must be acted on. Unlike
|
||||
* strcpybuff() it never aborts, so it suits text built from remote input.
|
||||
*/
|
||||
static HTS_INLINE HTS_UNUSED HTS_CHECK_RESULT HTS_PRINTF_FUN(3, 4) hts_boolean
|
||||
slprintfbuff(char *dest, size_t size, const char *fmt, ...) {
|
||||
va_list args;
|
||||
int ret;
|
||||
|
||||
assertf(dest != NULL && size != 0);
|
||||
va_start(args, fmt);
|
||||
ret = vsnprintf(dest, size, fmt, args);
|
||||
va_end(args);
|
||||
/* pre-C99 runtimes (msvcrt _vsnprintf) return -1 and do not terminate */
|
||||
dest[size - 1] = '\0';
|
||||
return ret >= 0 && (size_t) ret < size ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/**
|
||||
* slprintfbuff() over the in-scope array ARR (capacity = sizeof(ARR)).
|
||||
* On GCC/Clang a pointer is a compile error; use slprintfbuff() for those.
|
||||
*/
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
#define sprintfbuff(ARR, ...) \
|
||||
slprintfbuff((ARR), sizeof(ARR) + htsbuff_must_be_array_(ARR), __VA_ARGS__)
|
||||
#else
|
||||
#define sprintfbuff(ARR, ...) slprintfbuff((ARR), sizeof(ARR), __VA_ARGS__)
|
||||
#endif
|
||||
|
||||
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
|
||||
added bounds checking. freet() also NULLs the freed pointer and tolerates
|
||||
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the
|
||||
|
||||
@@ -51,7 +51,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscache_selftest.h"
|
||||
#include "htsdns_selftest.h"
|
||||
#include "htscharset.h"
|
||||
#include "htscmdline.h"
|
||||
#include "htsencoding.h"
|
||||
#include "htsftp.h"
|
||||
#include "htsmd5.h"
|
||||
@@ -59,7 +58,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscodec.h"
|
||||
#include "htsproxy.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
@@ -483,28 +481,6 @@ static int string_safety_selftests(void) {
|
||||
if (strcmp(buf, "abcd") != 0)
|
||||
return 1;
|
||||
|
||||
/* A decayed source has no known capacity, so the whole tail must land; a
|
||||
sizeof(char*) capacity would abort here instead. */
|
||||
{
|
||||
char src[32] = "0123456789abcdefghij";
|
||||
char dst[32];
|
||||
|
||||
strcpybuff(dst, src + 1);
|
||||
if (strcmp(dst, "123456789abcdefghij") != 0)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Truncating append: stops at N without aborting, what the status-message
|
||||
call sites rely on. */
|
||||
{
|
||||
char dst[10]; /* never sizeof(char*), or MSVC reads it as a pointer */
|
||||
|
||||
dst[0] = '\0';
|
||||
strncatbuff(dst, "abcdefghijkl", sizeof(dst) - 1);
|
||||
if (strcmp(dst, "abcdefghi") != 0)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* strlcpybuff: explicit-capacity copy into a pointer destination, the form
|
||||
the migration moves toward */
|
||||
{
|
||||
@@ -570,54 +546,6 @@ static int string_safety_selftests(void) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* sprintfbuff: truncate-and-report. Must never abort (its callers format
|
||||
remote banners) nor write past the array, which the canary catches. */
|
||||
{
|
||||
struct {
|
||||
char dst[8];
|
||||
char canary[8];
|
||||
} s;
|
||||
|
||||
const char *const big = "0123456789abcdefghijklmnopqrstuvwxyz";
|
||||
|
||||
/* repoison before every call, or an implementation that measures first and
|
||||
writes nothing "passes" the truncating cases on the previous content */
|
||||
#define POISON_DST() memset(s.dst, '#', sizeof(s.dst))
|
||||
|
||||
memset(&s, '#', sizeof(s));
|
||||
if (!sprintfbuff(s.dst, "%s-%d", "ab", 42) || strcmp(s.dst, "ab-42") != 0)
|
||||
return 1;
|
||||
|
||||
/* exact fit: 7 characters plus the NUL */
|
||||
POISON_DST();
|
||||
if (!sprintfbuff(s.dst, "%s", "1234567") || strcmp(s.dst, "1234567") != 0)
|
||||
return 1;
|
||||
|
||||
/* one over, then far over: truncated to the prefix, terminated, reported */
|
||||
POISON_DST();
|
||||
if (sprintfbuff(s.dst, "%s", "12345678") || strcmp(s.dst, "1234567") != 0)
|
||||
return 1;
|
||||
POISON_DST();
|
||||
if (sprintfbuff(s.dst, "%s", big) || strcmp(s.dst, "0123456") != 0)
|
||||
return 1;
|
||||
|
||||
/* explicit-capacity form, down to the degenerate size 1 */
|
||||
{
|
||||
char *const p = s.dst;
|
||||
|
||||
POISON_DST();
|
||||
if (slprintfbuff(p, 1, "%s", "x") || p[0] != '\0')
|
||||
return 1;
|
||||
POISON_DST();
|
||||
if (!slprintfbuff(p, sizeof(s.dst), "%s", "ok") || strcmp(p, "ok") != 0)
|
||||
return 1;
|
||||
}
|
||||
#undef POISON_DST
|
||||
|
||||
if (memcmp(s.canary, "########", sizeof(s.canary)) != 0)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* StringCatN/StringSetLength must eval SIZE once: (n_eval++, V) leaves
|
||||
n_eval == 2 on a double-eval macro. */
|
||||
{
|
||||
@@ -1176,128 +1104,6 @@ static int st_unescape_bounds(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// hts_split_cmdline(): the vector must grow with the argument count, and a
|
||||
// quote inside a value must not end the argument and hand -V to the parser.
|
||||
static int st_cmdlinesplit(httrackp *opt, int argc, char **argv) {
|
||||
char line[512];
|
||||
char **args;
|
||||
int nargs = 0;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
// control: every separator splits, and argv[0] is the program name
|
||||
strcpybuff(line, "httrack http://x/ --quiet\t-c8\n-O out");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(args != NULL && nargs == 6);
|
||||
assertf(args[nargs] == NULL); // callers may walk to the terminator
|
||||
assertf(strcmp(args[0], "httrack") == 0);
|
||||
assertf(strcmp(args[1], "http://x/") == 0);
|
||||
assertf(strcmp(args[2], "--quiet") == 0);
|
||||
assertf(strcmp(args[3], "-c8") == 0);
|
||||
assertf(strcmp(args[4], "-O") == 0);
|
||||
assertf(strcmp(args[5], "out") == 0);
|
||||
freet(args);
|
||||
|
||||
// the template pads with whitespace: empty arguments are kept (the engine
|
||||
// skips them), so the count is one per separator
|
||||
strcpybuff(line, "httrack --quiet");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 3 && args[1][0] == '\0');
|
||||
assertf(strcmp(args[2], "--quiet") == 0);
|
||||
freet(args);
|
||||
|
||||
// a quoted run keeps both its spaces and its quotes: the engine unquotes
|
||||
strcpybuff(line, "httrack --user-agent \"Mozilla 5.0\" -c8");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 4);
|
||||
assertf(strcmp(args[2], "\"Mozilla 5.0\"") == 0);
|
||||
assertf(strcmp(args[3], "-c8") == 0);
|
||||
freet(args);
|
||||
|
||||
// an escaped quote is a literal quote, not the end of the argument: the
|
||||
// engine strips only the outer pair
|
||||
strcpybuff(line, "httrack --user-agent \"x\\\" -V \\\"touch /tmp/pwn\" -c8");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 4);
|
||||
assertf(strcmp(args[2], "\"x\" -V \"touch /tmp/pwn\"") == 0);
|
||||
assertf(strcmp(args[3], "-c8") == 0);
|
||||
freet(args);
|
||||
|
||||
// \\ is a literal backslash, so a Windows path survives
|
||||
strcpybuff(line, "httrack --path \"C:\\\\dir\\\\sub\"");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 3);
|
||||
assertf(strcmp(args[2], "\"C:\\dir\\sub\"") == 0);
|
||||
freet(args);
|
||||
|
||||
// outside a quoted run a backslash is literal: the url and wildcard-filter
|
||||
// fields, which the wizard cannot quote, read as before
|
||||
strcpybuff(line, "httrack -*\\** +*.png");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 3);
|
||||
assertf(strcmp(args[1], "-*\\**") == 0);
|
||||
assertf(strcmp(args[2], "+*.png") == 0);
|
||||
freet(args);
|
||||
|
||||
// a quoted run leaves slots unused, so the terminator has to be written and
|
||||
// not inherited: size the vector from a full line first, so freeing it hands
|
||||
// the same chunk back with stale pointers in those slots
|
||||
strcpybuff(line, "httrack a b c d e");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 6);
|
||||
freet(args);
|
||||
strcpybuff(line, "httrack \"a b c d e\"");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 2);
|
||||
assertf(args[nargs] == NULL);
|
||||
freet(args);
|
||||
|
||||
// an unterminated quote protects the rest of the line, as one argument
|
||||
strcpybuff(line, "httrack --footer \"unbalanced -V x");
|
||||
args = hts_split_cmdline(line, &nargs);
|
||||
assertf(nargs == 3);
|
||||
assertf(strcmp(args[2], "\"unbalanced -V x") == 0);
|
||||
freet(args);
|
||||
|
||||
// past the 1024 entries the vector used to hold: distinct arguments, so a
|
||||
// write beyond the allocation cannot read back as the expected parse
|
||||
{
|
||||
const int n = 2000;
|
||||
const size_t size = 16 * (size_t) n + 16;
|
||||
char *big = malloct(size);
|
||||
size_t pos = 0;
|
||||
int i;
|
||||
|
||||
assertf(big != NULL);
|
||||
pos = (size_t) snprintf(big, size, "httrack");
|
||||
assertf(pos < size);
|
||||
for (i = 0; i < n; i++) {
|
||||
// snprintf returns what it wanted to write, so accumulating it blind
|
||||
// would let the next size argument wrap
|
||||
const int len = snprintf(big + pos, size - pos, " a%d", i);
|
||||
|
||||
assertf(len > 0 && (size_t) len < size - pos);
|
||||
pos += (size_t) len;
|
||||
}
|
||||
args = hts_split_cmdline(big, &nargs);
|
||||
assertf(args != NULL && nargs == n + 1);
|
||||
assertf(args[nargs] == NULL);
|
||||
for (i = 0; i < n; i++) {
|
||||
char expect[16];
|
||||
|
||||
snprintf(expect, sizeof(expect), "a%d", i);
|
||||
assertf(strcmp(args[i + 1], expect) == 0);
|
||||
}
|
||||
freet(args);
|
||||
freet(big);
|
||||
}
|
||||
|
||||
printf("cmdline-split self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_hashtable(httrackp *opt, int argc, char **argv) {
|
||||
char *snum;
|
||||
unsigned long count = 0;
|
||||
@@ -1493,14 +1299,6 @@ static int st_strsafe(httrackp *opt, int argc, char **argv) {
|
||||
htsbuff b = htsbuff_array(small);
|
||||
|
||||
htsbuff_cat(&b, src);
|
||||
} else if (strcmp(argv[0], "overflow-src") == 0) {
|
||||
/* Array source with no NUL: its capacity still comes from sizeof(), so
|
||||
the bounded strlen aborts rather than running off the array. */
|
||||
char nonul[6]; /* never sizeof(char*), per the note above */
|
||||
char big[64];
|
||||
|
||||
memset(nonul, src[0], sizeof(nonul));
|
||||
strcpybuff(big, nonul);
|
||||
} else {
|
||||
strcpybuff(small, src);
|
||||
}
|
||||
@@ -2195,77 +1993,6 @@ static int st_fsize(httrackp *opt, int argc, char **argv) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* 4GB+100KB wraps to ~108KB through an int, and needs 33 unsigned bits. A
|
||||
macro, not a static const: MSVC's C mode (/TC) rejects a const object
|
||||
used inside another object's static initializer below (C2099). */
|
||||
#define HTS_ST_GROWSIZE_OVER32 (4LL * 1024 * 1024 * 1024 + 100 * 1024)
|
||||
|
||||
/* llint_grow_size_t() sizes the buffer holding a whole -%S list file: the
|
||||
result must be the exact 64-bit sum or a clean refusal, never a short one. */
|
||||
static int st_growsize(httrackp *opt, int argc, char **argv) {
|
||||
enum { REFUSE, ACCEPT, WIDTH };
|
||||
|
||||
static const struct {
|
||||
size_t used;
|
||||
LLint extra;
|
||||
size_t slack;
|
||||
int want;
|
||||
} cases[] = {
|
||||
{0, 0, 0, ACCEPT},
|
||||
{10, 100, 8192, ACCEPT},
|
||||
{(size_t) -2 - 8, 4, 4, ACCEPT}, /* exact fit, no room to spare */
|
||||
{(size_t) -2, 0, 0, ACCEPT}, /* largest representable capacity */
|
||||
{0, -1, 0, REFUSE}, /* fsize() failure */
|
||||
/* -1 already maps to SIZE_MAX; only this exercises the negative guard */
|
||||
{0, -4096, 0, REFUSE},
|
||||
{(size_t) -1, 1, 0, REFUSE},
|
||||
{(size_t) -2, 0, 1, REFUSE}, /* slack alone overruns */
|
||||
{(size_t) -1 - 8, 4, 4, REFUSE}, /* total would be the error value */
|
||||
{(size_t) -1 - 8, 4, 8, REFUSE},
|
||||
{0, HTS_ST_GROWSIZE_OVER32, 8192,
|
||||
WIDTH}, /* 32-bit size_t can't hold these */
|
||||
{10, HTS_ST_GROWSIZE_OVER32, 8192, WIDTH},
|
||||
};
|
||||
|
||||
size_t k;
|
||||
int rc = 0;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
for (k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
|
||||
const size_t used = cases[k].used, slack = cases[k].slack;
|
||||
const LLint extra = cases[k].extra;
|
||||
const size_t got = llint_grow_size_t(used, extra, slack);
|
||||
const hts_boolean refused = got == (size_t) -1 ? HTS_TRUE : HTS_FALSE;
|
||||
const hts_boolean exact =
|
||||
!refused && extra >= 0 && got - used - slack == (size_t) extra;
|
||||
hts_boolean ok;
|
||||
|
||||
switch (cases[k].want) {
|
||||
case ACCEPT:
|
||||
ok = exact;
|
||||
break;
|
||||
case REFUSE:
|
||||
ok = refused;
|
||||
break;
|
||||
default:
|
||||
ok = sizeof(size_t) >= sizeof(LLint) ? exact : refused;
|
||||
break;
|
||||
}
|
||||
if (!ok) {
|
||||
fprintf(stderr,
|
||||
"growsize: grow(" LLintP ", " LLintP ", " LLintP ") = " LLintP
|
||||
" (want %s)\n",
|
||||
(LLint) used, extra, (LLint) slack, (LLint) got,
|
||||
cases[k].want == REFUSE ? "refusal" : "exact sum");
|
||||
rc = 1;
|
||||
}
|
||||
}
|
||||
printf("growsize self-test %s\n", rc == 0 ? "OK" : "FAILED");
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int st_savename(httrackp *opt, int argc, char **argv) {
|
||||
lien_adrfilsave afs;
|
||||
cache_back cache;
|
||||
@@ -2660,20 +2387,17 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
|
||||
static t_cookie ck2;
|
||||
htsblk r;
|
||||
char host[600];
|
||||
char line[64]; /* treathead NUL-cuts the header in place: never a literal */
|
||||
|
||||
memset(&r, 0, sizeof(r));
|
||||
memset(host, 'a', sizeof(host) - 1);
|
||||
host[sizeof(host) - 1] = '\0';
|
||||
ck2.max_len = (int) sizeof(ck2.data);
|
||||
ck2.data[0] = '\0';
|
||||
strcpybuff(line, "Set-Cookie: SID=1; path=/");
|
||||
treathead(&ck2, host, "/", &r, line);
|
||||
treathead(&ck2, host, "/", &r, "Set-Cookie: SID=1; path=/");
|
||||
if (strnotempty(ck2.data)) // oversize-host cookie was not dropped
|
||||
err = 1;
|
||||
/* control: a normal host still yields a cookie through treathead */
|
||||
strcpybuff(line, "Set-Cookie: SID=1; path=/");
|
||||
treathead(&ck2, dom, "/", &r, line);
|
||||
treathead(&ck2, dom, "/", &r, "Set-Cookie: SID=1; path=/");
|
||||
if (strstr(ck2.data, "SID") == NULL) // guard wrongly dropped a valid cookie
|
||||
err = 1;
|
||||
}
|
||||
@@ -2934,32 +2658,6 @@ static int st_makeindex(httrackp *opt, int argc, char **argv) {
|
||||
assertf(strstr(buf, "Refresh") != NULL);
|
||||
assertf(strstr(buf, "example.com") != NULL);
|
||||
|
||||
/* a first link whose escaped form overruns the old flat 1024-byte tempo: the
|
||||
redirect must carry the whole URL, not a clipped prefix */
|
||||
{
|
||||
char BIGSTK link[HTS_URLMAXSIZE * 2];
|
||||
char *p = link;
|
||||
|
||||
strcpybuff(link, "http://example.com/");
|
||||
p += strlen(link);
|
||||
memset(p, 'a', 1200);
|
||||
p += 1200;
|
||||
strcpy(p, "/end.html");
|
||||
|
||||
done = 0;
|
||||
fp = fopen(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
hts_finish_makeindex(opt, &done, &fp, 1, link, "%s%s", "", "");
|
||||
assertf(fp == NULL);
|
||||
fp = fopen(path, "rb");
|
||||
assertf(fp != NULL);
|
||||
n = fread(buf, 1, sizeof(buf) - 1, fp);
|
||||
fclose(fp);
|
||||
buf[n] = '\0';
|
||||
/* the closing quote proves the URL was not clipped mid-way */
|
||||
assertf(strstr(buf, "/end.html\">") != NULL);
|
||||
}
|
||||
|
||||
/* no single link: footer only, no refresh meta */
|
||||
done = 0;
|
||||
fp = fopen(path, "wb");
|
||||
@@ -3189,7 +2887,7 @@ static int ae_write_packed(const char *path, int windowBits,
|
||||
deflateEnd(&strm);
|
||||
return 1;
|
||||
}
|
||||
strm.next_in = (const Bytef *) src;
|
||||
strm.next_in = (Bytef *) src;
|
||||
strm.avail_in = (uInt) len;
|
||||
do {
|
||||
size_t n;
|
||||
@@ -4709,297 +4407,6 @@ static int st_warc_wacz(httrackp *opt, int argc, char **argv) {
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* --single-file */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
static int sf_err = 0;
|
||||
|
||||
static void sf_check(int ok, const char *what) {
|
||||
if (!ok) {
|
||||
fprintf(stderr, "singlefile: %s\n", what);
|
||||
sf_err++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Write rel (a '/'-separated path under dir), creating the directories. */
|
||||
static void sf_put(const char *dir, const char *rel, const void *data,
|
||||
size_t len) {
|
||||
char BIGSTK path[HTS_URLMAXSIZE * 2];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *fp;
|
||||
|
||||
fconcat(path, sizeof(path), dir, rel);
|
||||
structcheck_utf8(path);
|
||||
fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(len == 0 || fwrite(data, 1, len, fp) == len);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Number of times needle occurs in hay. */
|
||||
static int sf_count(const char *hay, const char *needle) {
|
||||
const size_t l = strlen(needle);
|
||||
int n = 0;
|
||||
const char *p = hay;
|
||||
|
||||
while ((p = strstr(p, needle)) != NULL) {
|
||||
n++;
|
||||
p += l;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* The base64 payload following the first occurrence of prefix, up to the first
|
||||
byte outside the base64 alphabet. NULL if prefix is absent. */
|
||||
static const char *sf_payload(const char *hay, const char *prefix,
|
||||
size_t *len) {
|
||||
const char *p = strstr(hay, prefix);
|
||||
size_t n = 0;
|
||||
|
||||
if (p == NULL)
|
||||
return NULL;
|
||||
p += strlen(prefix);
|
||||
while (p[n] != '\0' && (isalnum((unsigned char) p[n]) || p[n] == '+' ||
|
||||
p[n] == '/' || p[n] == '='))
|
||||
n++;
|
||||
*len = n;
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Independent base64 decoder: the round-trip check must not lean on code64().
|
||||
*/
|
||||
static unsigned char *sf_unb64(const char *s, size_t len, size_t *outlen) {
|
||||
static const char alpha[] =
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
unsigned char *out = (unsigned char *) malloct(len / 4 * 3 + 4);
|
||||
unsigned int acc = 0;
|
||||
size_t i, n = 0;
|
||||
int bits = 0;
|
||||
|
||||
if (out == NULL)
|
||||
return NULL;
|
||||
for (i = 0; i < len; i++) {
|
||||
const char *const p = s[i] != '\0' ? strchr(alpha, s[i]) : NULL;
|
||||
|
||||
if (s[i] == '=')
|
||||
break;
|
||||
if (p == NULL) {
|
||||
freet(out);
|
||||
return NULL;
|
||||
}
|
||||
acc = (acc << 6) | (unsigned int) (p - alpha);
|
||||
bits += 6;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out[n++] = (unsigned char) ((acc >> bits) & 0xff);
|
||||
}
|
||||
}
|
||||
*outlen = n;
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Decoded payload of the first data: URI with that MIME, as a NUL-terminated
|
||||
buffer the caller freet()s. NULL when absent or undecodable. */
|
||||
static char *sf_decode(const char *hay, const char *mime, size_t *outlen) {
|
||||
char prefix[128];
|
||||
size_t len = 0, dlen = 0;
|
||||
const char *b64;
|
||||
unsigned char *raw;
|
||||
|
||||
snprintf(prefix, sizeof(prefix), "data:%s;base64,", mime);
|
||||
b64 = sf_payload(hay, prefix, &len);
|
||||
if (b64 == NULL)
|
||||
return NULL;
|
||||
raw = sf_unb64(b64, len, &dlen);
|
||||
if (raw == NULL)
|
||||
return NULL;
|
||||
raw[dlen] = '\0';
|
||||
if (outlen != NULL)
|
||||
*outlen = dlen;
|
||||
return (char *) raw;
|
||||
}
|
||||
|
||||
/* The 12-byte asset: high bytes and an embedded NUL, so a text-shaped copy
|
||||
would be caught. */
|
||||
static const char sf_png[] = "\x89PNG\r\n\x1a\n\x00\x01\x02\xff";
|
||||
#define SF_PNG_LEN 12
|
||||
|
||||
static const char sf_page[] =
|
||||
"<html><head>\n"
|
||||
"<link rel=\"stylesheet\" href=\"css/main.css\">\n"
|
||||
"<link rel=\"canonical\" href=\"other.html\">\n"
|
||||
"<title>t</title>\n"
|
||||
"<style>body { background: url(\"img/a%20b.png\"); }</style>\n"
|
||||
"</head><body>\n"
|
||||
"<img src=\"img/a%20b.png\" srcset=\"img/a%20b.png 1x, img/big.png 2x\">\n"
|
||||
"<img src=\"data:image/gif;base64,QUJD\">\n"
|
||||
"<img src=\"http://example.com/x.png\">\n"
|
||||
/* Two shapes of climbing out of the mirror, each aimed at a real image so
|
||||
a missing clamp inlines rather than merely failing to find a file: one
|
||||
resolves outside the root, one lands back inside it if a leading ".."
|
||||
is silently dropped. */
|
||||
"<img src=\"../escape.png\">\n"
|
||||
"<img src=\"../img/a%20b.png\">\n"
|
||||
"<a href=\"img/a%20b.png\">link</a>\n"
|
||||
"<video controls><source src=\"v.mp4\" type=\"video/mp4\"></video>\n"
|
||||
"<script src=\"js/app.js\"></script>\n"
|
||||
"<script>var s = \"<img src='img/a%20b.png'>\";</script>\n"
|
||||
"<div style=\"background:url(img/a%20b.png)\"></div>\n"
|
||||
"<div style='content:\"x\"; background:url(img/a%20b.png)'></div>\n"
|
||||
"</body></html>\n";
|
||||
|
||||
/* Lay a small mirror down under root and return the page path in page. */
|
||||
static void sf_fixture(const char *root, char *page, size_t pagesize) {
|
||||
static const char css[] = "@import \"sub/nested.css\";\n"
|
||||
"body { background: url(../img/a b.png); }\n"
|
||||
"/* url(../img/never.png) */\n";
|
||||
static const char nested[] = "div { background: url(../../img/a b.png); }\n";
|
||||
static const char js[] = "var app = 1;\n";
|
||||
char big[4096];
|
||||
|
||||
memset(big, 'B', sizeof(big));
|
||||
sf_put(root, "page.html", sf_page, sizeof(sf_page) - 1);
|
||||
sf_put(root, "other.html", "<html>o</html>", 14);
|
||||
sf_put(root, "css/main.css", css, sizeof(css) - 1);
|
||||
sf_put(root, "css/sub/nested.css", nested, sizeof(nested) - 1);
|
||||
sf_put(root, "js/app.js", js, sizeof(js) - 1);
|
||||
sf_put(root, "img/a b.png", sf_png, SF_PNG_LEN);
|
||||
sf_put(root, "img/big.png", big, sizeof(big));
|
||||
sf_put(root, "v.mp4",
|
||||
"\x00\x00\x00\x18"
|
||||
"ftypisom",
|
||||
12);
|
||||
fconcat(page, pagesize, root, "page.html");
|
||||
}
|
||||
|
||||
/* -#test=singlefile <dir>: rewrite a hand-built mirror and check what gets
|
||||
inlined, what must keep its link, the per-asset cap, and idempotence. */
|
||||
static int st_singlefile(httrackp *opt, int argc, char **argv) {
|
||||
char BIGSTK root[HTS_URLMAXSIZE];
|
||||
char BIGSTK page[HTS_URLMAXSIZE * 2];
|
||||
const LLint saved_cap = opt->single_file_max_size;
|
||||
char *out, *css, *nested;
|
||||
size_t outlen = 0, len = 0;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "singlefile: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
sf_err = 0;
|
||||
sf_put(argv[0], "escape.png", sf_png,
|
||||
SF_PNG_LEN); /* just outside the mirror */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror/");
|
||||
sf_fixture(root, page, sizeof(page));
|
||||
|
||||
/* Cap between the small assets and big.png. */
|
||||
opt->single_file_max_size = 1024;
|
||||
sf_check(singlefile_rewrite_file(opt, root, page),
|
||||
"first pass changed nothing");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
|
||||
/* Inlined, and the payload is the file's exact bytes. */
|
||||
{
|
||||
char *img = sf_decode(out, "image/png", &len);
|
||||
|
||||
sf_check(img != NULL && len == SF_PNG_LEN &&
|
||||
memcmp(img, sf_png, SF_PNG_LEN) == 0,
|
||||
"image payload does not round-trip");
|
||||
freet(img);
|
||||
}
|
||||
sf_check(strstr(out, "data:application/x-javascript;base64,") != NULL,
|
||||
"script not inlined");
|
||||
sf_check(strstr(out, "href=\"css/main.css\"") == NULL,
|
||||
"stylesheet not inlined");
|
||||
|
||||
/* Left alone: a page link, a navigational <link>, media, an absolute URL, an
|
||||
existing data: URI, and a reference climbing out of the mirror. */
|
||||
sf_check(strstr(out, "<a href=\"img/a%20b.png\">") != NULL, "anchor inlined");
|
||||
sf_check(strstr(out, "href=\"other.html\"") != NULL, "rel=canonical inlined");
|
||||
sf_check(strstr(out, "src=\"v.mp4\"") != NULL, "video source inlined");
|
||||
sf_check(strstr(out, "src=\"http://example.com/x.png\"") != NULL,
|
||||
"absolute URL rewritten");
|
||||
sf_check(sf_count(out, "QUJD") == 1, "existing data: URI not preserved");
|
||||
sf_check(strstr(out, "src=\"../escape.png\"") != NULL,
|
||||
"a reference outside the mirror was resolved");
|
||||
sf_check(strstr(out, "src=\"../img/a%20b.png\"") != NULL,
|
||||
"a leading .. was dropped instead of rejected");
|
||||
sf_check(strstr(out, "var s = \"<img src='img/a%20b.png'>\";") != NULL,
|
||||
"script body rewritten");
|
||||
|
||||
/* Nothing an attribute value cannot hold: url() stays unquoted, and a quote
|
||||
that was already in the CSS is escaped. */
|
||||
sf_check(strstr(out, "url(\"data:") == NULL,
|
||||
"a quoted url() would end a style attribute");
|
||||
sf_check(strstr(out, "style=\"content:"x"; background:url(data:") !=
|
||||
NULL,
|
||||
"quote inside a rewritten style attribute not escaped");
|
||||
|
||||
/* The over-cap srcset candidate keeps its link, the small one does not. */
|
||||
sf_check(strstr(out, "img/big.png 2x") != NULL, "over-cap asset inlined");
|
||||
sf_check(strstr(out, " 1x") != NULL, "srcset descriptor lost");
|
||||
sf_check(sf_count(out, "img/a%20b.png") ==
|
||||
3, /* anchor, script, the ".." one */
|
||||
"an inlinable reference was left as a link");
|
||||
|
||||
/* The inlined stylesheet carries its own @import and url() inlined. */
|
||||
css = sf_decode(out, "text/css", NULL);
|
||||
sf_check(css != NULL, "stylesheet payload undecodable");
|
||||
if (css != NULL) {
|
||||
sf_check(strstr(css, "@import \"data:text/css;base64,") != NULL,
|
||||
"@import not inlined");
|
||||
sf_check(strstr(css, "url(data:image/png;base64,") != NULL,
|
||||
"url() in stylesheet not inlined");
|
||||
sf_check(strstr(css, "url(../img/never.png)") != NULL,
|
||||
"url() inside a CSS comment was rewritten");
|
||||
nested = sf_decode(css, "text/css", NULL);
|
||||
sf_check(nested != NULL &&
|
||||
strstr(nested, "url(data:image/png;base64,") != NULL,
|
||||
"url() in the @import'ed stylesheet not inlined");
|
||||
freet(nested);
|
||||
freet(css);
|
||||
}
|
||||
|
||||
/* Idempotence: a second pass must find nothing and leave the bytes alone. */
|
||||
sf_check(!singlefile_rewrite_file(opt, root, page), "second pass rewrote");
|
||||
{
|
||||
char *again = readfile_utf8(page);
|
||||
|
||||
sf_check(again != NULL && strcmp(again, out) == 0,
|
||||
"second pass changed the page");
|
||||
freet(again);
|
||||
}
|
||||
freet(out);
|
||||
|
||||
/* Same page, a cap above big.png: it now inlines. */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror2/");
|
||||
sf_fixture(root, page, sizeof(page));
|
||||
opt->single_file_max_size = 1024 * 1024;
|
||||
sf_check(singlefile_rewrite_file(opt, root, page),
|
||||
"raised-cap pass changed nothing");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
sf_check(strstr(out, "img/big.png 2x") == NULL,
|
||||
"asset under the raised cap still a link");
|
||||
freet(out);
|
||||
|
||||
/* A cap of one byte inlines nothing at all. */
|
||||
fconcat(root, sizeof(root), argv[0], "mirror3/");
|
||||
sf_fixture(root, page, sizeof(page));
|
||||
opt->single_file_max_size = 1;
|
||||
sf_check(!singlefile_rewrite_file(opt, root, page), "one-byte cap inlined");
|
||||
out = readfile_utf8(page);
|
||||
assertf(out != NULL);
|
||||
sf_check(strcmp(out, sf_page) == 0, "one-byte cap altered the page");
|
||||
freet(out);
|
||||
(void) outlen;
|
||||
|
||||
opt->single_file_max_size = saved_cap;
|
||||
printf("singlefile: %s\n", sf_err ? "FAIL" : "OK");
|
||||
return sf_err;
|
||||
}
|
||||
|
||||
// -#test=longpath <dir>: round-trip a >MAX_PATH (260) file through the file
|
||||
// wrappers, exercising hts_pathToUCS2's \\?\ prefixing on Windows (#133).
|
||||
static int st_longpath(httrackp *opt, int argc, char **argv) {
|
||||
@@ -5399,12 +4806,9 @@ static const struct selftest_entry {
|
||||
st_footerfmt},
|
||||
{"unescape-bounds", "", "unescapers reserve the NUL byte (no 1-byte OOB)",
|
||||
st_unescape_bounds},
|
||||
{"cmdline-split", "",
|
||||
"webhttrack command-line to argv split (bounds, quoting)",
|
||||
st_cmdlinesplit},
|
||||
{"hashtable", "<count|file>", "coucal hashtable stress test", st_hashtable},
|
||||
{"strsafe", "[overflow|overflow-buff|overflow-src [str]]",
|
||||
"bounded string-op self-test", st_strsafe},
|
||||
{"strsafe", "[overflow|overflow-buff [str]]", "bounded string-op self-test",
|
||||
st_strsafe},
|
||||
{"copyopt", "", "copy_htsopt option-copy self-test", st_copyopt},
|
||||
{"pause", "", "randomized inter-file pause target self-test", st_pause},
|
||||
{"relative", "<link> <curr-file>", "relative link between two paths",
|
||||
@@ -5434,8 +4838,6 @@ static const struct selftest_entry {
|
||||
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
|
||||
st_sniff},
|
||||
{"fsize", "<dir>", "file size past the 2GB signed-32-bit wrap", st_fsize},
|
||||
{"growsize", "", "buffer capacity for a 64-bit file size (no int wrap)",
|
||||
st_growsize},
|
||||
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
|
||||
{"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
|
||||
st_cacheindex},
|
||||
@@ -5508,9 +4910,6 @@ static const struct selftest_entry {
|
||||
{"warc-wacz", "<dir>", "--wacz package: layout, STORE mode, sha256 digests",
|
||||
st_warc_wacz},
|
||||
#endif
|
||||
{"singlefile", "<dir>",
|
||||
"--single-file: what is inlined, the per-asset cap, idempotence",
|
||||
st_singlefile},
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
251
src/htsserver.c
251
src/htsserver.c
@@ -147,8 +147,7 @@ HTS_UNUSED static int LANG_LIST(const char *path, char *buffer, size_t size);
|
||||
// 0- Init the URL catcher with standard port
|
||||
|
||||
// smallserver_init(&port,&return_host);
|
||||
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort,
|
||||
const char *bindAddr) {
|
||||
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort) {
|
||||
T_SOC soc;
|
||||
|
||||
if (defaultPort <= 0) {
|
||||
@@ -161,12 +160,12 @@ T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort,
|
||||
int i = 0;
|
||||
|
||||
do {
|
||||
soc = smallserver_init(&try_to_listen_to[i], adr_prox, bindAddr);
|
||||
soc = smallserver_init(&try_to_listen_to[i], adr_prox);
|
||||
*port_prox = try_to_listen_to[i];
|
||||
i++;
|
||||
} while((soc == INVALID_SOCKET) && (try_to_listen_to[i] >= 0));
|
||||
} else {
|
||||
soc = smallserver_init(&defaultPort, adr_prox, bindAddr);
|
||||
soc = smallserver_init(&defaultPort, adr_prox);
|
||||
*port_prox = defaultPort;
|
||||
}
|
||||
return soc;
|
||||
@@ -244,10 +243,9 @@ static int my_gethostname(char *h_loc, size_t size) {
|
||||
}
|
||||
|
||||
// smallserver_init(&port,&return_host);
|
||||
T_SOC smallserver_init(int *port, char *adr, const char *bindAddr) {
|
||||
T_SOC smallserver_init(int *port, char *adr) {
|
||||
T_SOC soc = INVALID_SOCKET;
|
||||
char h_loc[256 + 2];
|
||||
SOCaddr server;
|
||||
|
||||
commandRunning = commandEnd = commandReturn = commandReturnSet =
|
||||
commandEndRequested = 0;
|
||||
@@ -258,23 +256,25 @@ T_SOC smallserver_init(int *port, char *adr, const char *bindAddr) {
|
||||
free(commandReturnCmdl);
|
||||
commandReturnCmdl = NULL;
|
||||
|
||||
SOCaddr_initany(server);
|
||||
if (bindAddr != NULL && *bindAddr != '\0') {
|
||||
/* advertise the bound address, else the URL we print is unreachable */
|
||||
if (strlen(bindAddr) >= sizeof(h_loc) || !gethost(bindAddr, &server)) {
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
strcpybuff(h_loc, bindAddr);
|
||||
} else if (my_gethostname(h_loc, 256) != 0) { // host name
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
if (my_gethostname(h_loc, 256) == 0) { // host name
|
||||
SOCaddr server;
|
||||
|
||||
if ((soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0)) !=
|
||||
INVALID_SOCKET) {
|
||||
SOCaddr_initport(server, *port);
|
||||
if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
|
||||
if (listen(soc, 10) >= 0) {
|
||||
strcpy(adr, h_loc);
|
||||
SOCaddr_initany(server);
|
||||
if ((soc =
|
||||
(T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM,
|
||||
0)) != INVALID_SOCKET) {
|
||||
SOCaddr_initport(server, *port);
|
||||
if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
|
||||
if (listen(soc, 10) >= 0) {
|
||||
strcpy(adr, h_loc);
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
closesocket(soc);
|
||||
#else
|
||||
close(soc);
|
||||
#endif
|
||||
soc = INVALID_SOCKET;
|
||||
}
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
closesocket(soc);
|
||||
@@ -283,13 +283,6 @@ T_SOC smallserver_init(int *port, char *adr, const char *bindAddr) {
|
||||
#endif
|
||||
soc = INVALID_SOCKET;
|
||||
}
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
closesocket(soc);
|
||||
#else
|
||||
close(soc);
|
||||
#endif
|
||||
soc = INVALID_SOCKET;
|
||||
}
|
||||
}
|
||||
return soc;
|
||||
@@ -318,88 +311,6 @@ typedef struct {
|
||||
error_redirect = "/server/error.html"; \
|
||||
} while(0)
|
||||
|
||||
/* Longest "sid" value worth unescaping: the expected one is an md5 hex digest,
|
||||
so anything near this is already invalid and is rejected unread. */
|
||||
#define SID_VALUE_MAX 64
|
||||
|
||||
/** Does the urlencoded request body present the expected session id?
|
||||
True only if at least one "sid" field is present and every occurrence
|
||||
matches, so it holds whichever one a later last-write-wins parse keeps.
|
||||
Non-destructive: it runs before the body is tokenized in place. */
|
||||
static hts_boolean body_sid_is_valid(const char *body, const char *expected) {
|
||||
const char *s = body;
|
||||
hts_boolean seen = HTS_FALSE;
|
||||
|
||||
while (s != NULL && *s != '\0') {
|
||||
const char *const amp = strchr(s, '&');
|
||||
const char *const eq = strchr(s, '=');
|
||||
|
||||
if (eq != NULL && (amp == NULL || eq < amp) && (size_t) (eq - s) == 3 &&
|
||||
strncmp(s, "sid", 3) == 0) {
|
||||
const size_t len = amp != NULL ? (size_t) (amp - eq - 1) : strlen(eq + 1);
|
||||
hts_boolean match = HTS_FALSE;
|
||||
|
||||
if (len < SID_VALUE_MAX) {
|
||||
char raw[SID_VALUE_MAX];
|
||||
String value = STRING_EMPTY;
|
||||
|
||||
memcpy(raw, eq + 1, len);
|
||||
raw[len] = '\0';
|
||||
unescapehttp(raw, &value);
|
||||
/* StringBuff is NULL until written, so an empty value lands here. */
|
||||
if (StringBuff(value) != NULL &&
|
||||
strcmp(StringBuff(value), expected) == 0) {
|
||||
match = HTS_TRUE;
|
||||
}
|
||||
StringFree(value);
|
||||
}
|
||||
if (!match) {
|
||||
return HTS_FALSE;
|
||||
}
|
||||
seen = HTS_TRUE;
|
||||
}
|
||||
s = amp != NULL ? amp + 1 : NULL;
|
||||
}
|
||||
return seen;
|
||||
}
|
||||
|
||||
/* Append c to dst as an HTML entity, or return HTS_FALSE if it needs none. */
|
||||
static hts_boolean cat_html_escaped(String *dst, char c) {
|
||||
switch (c) {
|
||||
case '<':
|
||||
StringCat(*dst, "<");
|
||||
break;
|
||||
case '>':
|
||||
StringCat(*dst, ">");
|
||||
break;
|
||||
case '&':
|
||||
StringCat(*dst, "&");
|
||||
break;
|
||||
case '\'':
|
||||
StringCat(*dst, "'");
|
||||
break;
|
||||
default:
|
||||
return HTS_FALSE;
|
||||
}
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Append the value of a double-quoted command-line argument: escaped for HTML,
|
||||
which the browser undoes when it posts the command line back, and for the
|
||||
argv splitter, which does not. */
|
||||
static void cat_cmdline_arg(String *output, const char *value) {
|
||||
const char *a;
|
||||
|
||||
for (a = value; *a != '\0'; a++) {
|
||||
if (*a == '\\' || *a == '\"') {
|
||||
StringCat(*output, "\\");
|
||||
}
|
||||
if (!cat_html_escaped(output, *a)) {
|
||||
StringMemcat(*output, a, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
int timeout = 30;
|
||||
int retour = 0;
|
||||
@@ -484,7 +395,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
T_SOC soc_c;
|
||||
LLint length = 0;
|
||||
const char *error_redirect = NULL;
|
||||
hts_boolean denied = HTS_FALSE;
|
||||
|
||||
line[0] = '\0';
|
||||
buffer[0] = '\0';
|
||||
@@ -596,22 +506,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Authenticate the body before parsing it: every field it carries is
|
||||
written straight into the global key store below, "command" included,
|
||||
and that one reaches the engine. Checking afterwards cannot work — the
|
||||
damage is already done, and the pre-seeded "sid" above would compare
|
||||
equal to itself for a request that simply omits the field. */
|
||||
if (meth && buffer[0]) {
|
||||
intptr_t expected = 0;
|
||||
|
||||
if (!coucal_readptr(NewLangList, "_sid", &expected) ||
|
||||
!body_sid_is_valid(buffer, (const char *) expected)) {
|
||||
buffer[0] = '\0';
|
||||
meth = 0;
|
||||
denied = HTS_TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
/* check variables */
|
||||
if (meth && buffer[0]) {
|
||||
char *s = buffer;
|
||||
@@ -632,6 +526,20 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Error check */
|
||||
{
|
||||
intptr_t adr = 0;
|
||||
intptr_t adr2 = 0;
|
||||
|
||||
if (coucal_readptr(NewLangList, "sid", &adr)) {
|
||||
if (coucal_readptr(NewLangList, "_sid", &adr2)) {
|
||||
if (strcmp((char *) adr, (char *) adr2) != 0) {
|
||||
meth = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Check variables (internal) */
|
||||
if (meth) {
|
||||
int doLoad = 0;
|
||||
@@ -894,7 +802,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
|
||||
/* Response */
|
||||
if (meth) {
|
||||
hts_boolean virtualpath = HTS_FALSE;
|
||||
int virtualpath = 0;
|
||||
char *pos;
|
||||
char *url = strchr(line1, ' ');
|
||||
|
||||
@@ -922,7 +830,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
|
||||
if (strncmp(file, "/website/", 9) == 0) {
|
||||
virtualpath = HTS_TRUE;
|
||||
virtualpath = 1;
|
||||
}
|
||||
|
||||
/* override */
|
||||
@@ -936,11 +844,8 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
}
|
||||
|
||||
/* the override above may have swapped a mirror path for a GUI page */
|
||||
virtualpath = strncmp(file, "/website/", 9) == 0;
|
||||
|
||||
if (strlen(path) + strlen(file) + 32 < sizeof(fsfile)) {
|
||||
if (!virtualpath) {
|
||||
if (strncmp(file, "/website/", 9) != 0) {
|
||||
sprintf(fsfile, "%shtml%s", path, file);
|
||||
} else {
|
||||
intptr_t adr = 0;
|
||||
@@ -998,16 +903,16 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
}
|
||||
StringMemcat(headers, redir, strlen(redir));
|
||||
/* client-supplied: a CR/LF here would split the response */
|
||||
if (newfile[strcspn(newfile, "\r\n")] == '\0') {
|
||||
StringCat(headers, "Location: ");
|
||||
StringCat(headers, newfile);
|
||||
StringCat(headers, "\r\n");
|
||||
{
|
||||
char tmp[256];
|
||||
|
||||
if (strlen(file) < sizeof(tmp) - 32) {
|
||||
sprintf(tmp, "Location: %s\r\n", newfile);
|
||||
StringMemcat(headers, tmp, strlen(tmp));
|
||||
}
|
||||
}
|
||||
coucal_write(NewLangList, "redirect", (intptr_t) NULL);
|
||||
} else if (!virtualpath && is_html(file)) {
|
||||
/* GUI templates only: ${_sid} in a mirrored page would hand the
|
||||
crawled site the session id that authenticates commands */
|
||||
} else if (is_html(file)) {
|
||||
int outputmode = 0;
|
||||
|
||||
StringMemcat(headers, ok, sizeof(ok) - 1);
|
||||
@@ -1035,7 +940,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
int p;
|
||||
int format = 0;
|
||||
int listDefault = 0;
|
||||
hts_boolean unquoted = HTS_FALSE;
|
||||
|
||||
name[0] = '\0';
|
||||
strlncatbuff(name, str, sizeof(name_), n);
|
||||
@@ -1045,12 +949,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
} else if ((p = strfield(name, "html:"))) {
|
||||
name += p;
|
||||
format = 1;
|
||||
} else if ((p = strfield(name, "unquoted:"))) {
|
||||
name += p;
|
||||
unquoted = HTS_TRUE;
|
||||
} else if ((p = strfield(name, "arg:"))) {
|
||||
name += p;
|
||||
format = 5;
|
||||
} else if ((p = strfield(name, "list:"))) {
|
||||
name += p;
|
||||
format = 2;
|
||||
@@ -1187,8 +1085,8 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
test:<if ==0>:<if ==1>:<if == 2>..
|
||||
ztest:<if == 0 || !exist>:<if == 1>:<if == 2>..
|
||||
*/
|
||||
else if ((p = strfield(name, "test:")) ||
|
||||
(p = strfield(name, "ztest:"))) {
|
||||
else if ((p = strfield(name, "test:"))
|
||||
|| (p = strfield(name, "ztest:"))) {
|
||||
intptr_t adr = 0;
|
||||
char *pos2;
|
||||
int ztest = (name[0] == 'z');
|
||||
@@ -1291,12 +1189,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
}
|
||||
}
|
||||
/* consumed here: it shares nothing with the list and
|
||||
option formats below */
|
||||
if (format == 5 && langstr != NULL && outputmode != -1) {
|
||||
cat_cmdline_arg(&output, langstr);
|
||||
langstr = NULL;
|
||||
}
|
||||
if (langstr && outputmode != -1) {
|
||||
switch (format) {
|
||||
case 0:
|
||||
@@ -1314,18 +1206,18 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
StringMemcat(output, &c, 1);
|
||||
}
|
||||
a += 2;
|
||||
} else if (unquoted && a[0] == '\"') {
|
||||
/* the browser posts an entity back as a raw
|
||||
quote, which would open a quoted run in the
|
||||
argv splitter; a URI cannot hold one anyway */
|
||||
StringCat(output, "%22");
|
||||
} else if (outputmode &&
|
||||
cat_html_escaped(&output, a[0])) {
|
||||
/* appended as an entity */
|
||||
} else if (outputmode && a[0] == '<') {
|
||||
StringCat(output, "<");
|
||||
} else if (outputmode && a[0] == '>') {
|
||||
StringCat(output, ">");
|
||||
} else if (outputmode && a[0] == '&') {
|
||||
StringCat(output, "&");
|
||||
} else if (outputmode && a[0] == '\'') {
|
||||
StringCat(output, "'");
|
||||
} else if (outputmode == 3 && a[0] == ' ') {
|
||||
StringCat(output, "%20");
|
||||
} else if (outputmode >= 2 &&
|
||||
((unsigned char) a[0]) < 32) {
|
||||
} else if (outputmode >= 2
|
||||
&& ((unsigned char) a[0]) < 32) {
|
||||
char tmp[32];
|
||||
|
||||
sprintf(tmp, "%%%02x", (unsigned char) a[0]);
|
||||
@@ -1386,10 +1278,20 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
StringClear(tmpbuff);
|
||||
break;
|
||||
case '<':
|
||||
StringCat(tmpbuff, "<");
|
||||
break;
|
||||
case '>':
|
||||
StringCat(tmpbuff, ">");
|
||||
break;
|
||||
case '&':
|
||||
StringCat(tmpbuff, "&");
|
||||
break;
|
||||
case '\'':
|
||||
StringCat(tmpbuff, "'");
|
||||
break;
|
||||
default:
|
||||
if (!cat_html_escaped(&tmpbuff, *fstr)) {
|
||||
StringMemcat(tmpbuff, fstr, 1);
|
||||
}
|
||||
StringMemcat(tmpbuff, fstr, 1);
|
||||
break;
|
||||
}
|
||||
fstr++;
|
||||
@@ -1429,9 +1331,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
if (is_html(file)) {
|
||||
StringMemcat(headers, ok, sizeof(ok) - 1);
|
||||
} else if (is_text(file)) {
|
||||
if (is_text(file)) {
|
||||
StringMemcat(headers, ok_text, sizeof(ok_text) - 1);
|
||||
} else if (is_js(file)) {
|
||||
StringMemcat(headers, ok_js, sizeof(ok_js) - 1);
|
||||
@@ -1468,11 +1368,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
StringCat(output, error);
|
||||
}
|
||||
}
|
||||
} else if (denied) {
|
||||
StringCat(headers, "HTTP/1.0 403 Forbidden\r\n"
|
||||
"Server: httrack small server\r\n"
|
||||
"Content-type: text/html\r\n");
|
||||
StringCat(output, "Missing or invalid session id.\r\n");
|
||||
} else {
|
||||
#ifdef _DEBUG
|
||||
char error_hdr[] =
|
||||
|
||||
@@ -43,11 +43,8 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
// Fonctions
|
||||
void socinput(T_SOC soc, char *s, int max);
|
||||
/* Listen on bindAddr, or every interface if NULL/empty; adr (>= 258 bytes) gets
|
||||
the address to advertise. INVALID_SOCKET on error. */
|
||||
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort,
|
||||
const char *bindAddr);
|
||||
T_SOC smallserver_init(int *port, char *adr, const char *bindAddr);
|
||||
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort);
|
||||
T_SOC smallserver_init(int *port, char *adr);
|
||||
int smallserver(T_SOC soc, char *url, char *method, char *data, char *path);
|
||||
|
||||
#define CATCH_RESPONSE \
|
||||
|
||||
1076
src/htssinglefile.c
1076
src/htssinglefile.c
File diff suppressed because it is too large
Load Diff
@@ -1,72 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* --single-file: post-mirror pass embedding each page's assets as data: URIs.
|
||||
Internal, not installed. Unlike -%M (one MHT archive for the whole mirror)
|
||||
this rewrites the saved pages in place and keeps page-to-page links
|
||||
relative, so the mirror stays browsable and every page also stands alone. */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_SINGLEFILE_DEFH
|
||||
#define HTS_SINGLEFILE_DEFH
|
||||
|
||||
#include "htsopt.h"
|
||||
#include "htsstrings.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* --single-file-max-size default: a bigger asset stays an ordinary link. */
|
||||
#define SINGLEFILE_DEFAULT_MAX_SIZE (10 * 1024 * 1024)
|
||||
|
||||
/* Never base64 more than this, whatever the cap: code64() sizes in int. */
|
||||
#define SINGLEFILE_HARD_MAX_SIZE (256 * 1024 * 1024)
|
||||
|
||||
/* Rewrite every HTML page the mirror produced. No-op unless opt->single_file;
|
||||
call once the tree is final (after the update purge and the index). */
|
||||
void singlefile_process_mirror(httrackp *opt);
|
||||
|
||||
/* Rewrite one HTML document held in memory, appending the result to out.
|
||||
root is the mirror directory that references may not escape; page_path is
|
||||
the document's own path under it (both UTF-8, '/' or native separators).
|
||||
Returns HTS_TRUE if at least one reference was replaced. */
|
||||
hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
|
||||
const char *page_path, const char *html,
|
||||
size_t html_len, String *out);
|
||||
|
||||
/* singlefile_rewrite_html() over the file at page_path, rewritten in place.
|
||||
Returns HTS_TRUE if the file changed. */
|
||||
hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
|
||||
const char *page_path);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -1318,19 +1318,19 @@ HTSEXT_API hts_boolean hts_findnext(find_handle find) {
|
||||
if (find) {
|
||||
#ifdef _WIN32
|
||||
if ((FindNextFileA(find->handle, &find->hdata)))
|
||||
return HTS_TRUE;
|
||||
return 1;
|
||||
#else
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
|
||||
memset(&(find->filestat), 0, sizeof(find->filestat));
|
||||
if ((find->dirp = readdir(find->hdir)))
|
||||
if (!STAT(
|
||||
concat(catbuff, sizeof(catbuff), find->path, find->dirp->d_name),
|
||||
&find->filestat))
|
||||
return HTS_TRUE;
|
||||
if (find->dirp->d_name)
|
||||
if (!STAT
|
||||
(concat(catbuff, sizeof(catbuff), find->path, find->dirp->d_name), &find->filestat))
|
||||
return 1;
|
||||
#endif
|
||||
}
|
||||
return HTS_FALSE;
|
||||
return 0;
|
||||
}
|
||||
|
||||
HTSEXT_API int hts_findclose(find_handle find) {
|
||||
|
||||
66
src/htsweb.c
66
src/htsweb.c
@@ -62,7 +62,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsmd5.c"
|
||||
#include "md5.c"
|
||||
|
||||
#include "htscmdline.h"
|
||||
#include "htsserver.h"
|
||||
#include "htsurlport.h"
|
||||
#include "htsweb.h"
|
||||
@@ -89,7 +88,7 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
static htsmutex refreshMutex = HTSMUTEX_INIT;
|
||||
|
||||
static int help_server(char *dest_path, int defaultPort, const char *bindAddr);
|
||||
static int help_server(char *dest_path, int defaultPort);
|
||||
extern int commandRunning;
|
||||
extern int commandEnd;
|
||||
extern int commandReturn;
|
||||
@@ -154,8 +153,6 @@ int main(int argc, char *argv[]) {
|
||||
int ret = 0;
|
||||
int defaultPort = 0;
|
||||
int parentPid = 0;
|
||||
/* loopback by default: the handler trusts its input; --bind widens it */
|
||||
const char *bindAddr = "127.0.0.1";
|
||||
|
||||
printf("Initializing the server..\n");
|
||||
|
||||
@@ -182,8 +179,7 @@ int main(int argc, char *argv[]) {
|
||||
if (argc < 2 || (argc % 2) != 0) {
|
||||
fprintf(stderr, "** Warning: use the webhttrack frontend if available\n");
|
||||
fprintf(stderr,
|
||||
"usage: %s [--port <port>] [--bind <address>] [--ppid parent-pid] "
|
||||
"<path-to-html-root-dir> [key value [key value]..]\n",
|
||||
"usage: %s [--port <port>] [--ppid parent-pid] <path-to-html-root-dir> [key value [key value]..]\n",
|
||||
argv[0]);
|
||||
fprintf(stderr, "example: %s /usr/share/httrack/\n", argv[0]);
|
||||
return 1;
|
||||
@@ -271,14 +267,6 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "couldn't set the port number to %s\n", argv[i + 1]);
|
||||
return -1;
|
||||
}
|
||||
} else if (strcmp(argv[i], "--bind") == 0 && i + 1 < argc) {
|
||||
/* empty would fall back to every interface, silently undoing the default
|
||||
*/
|
||||
if (!strnotempty(argv[i + 1])) {
|
||||
fprintf(stderr, "--bind needs an address\n");
|
||||
return -1;
|
||||
}
|
||||
bindAddr = argv[i + 1];
|
||||
} else if (strcmp(argv[i], "--ppid") == 0 && i + 1 < argc) {
|
||||
if (sscanf(argv[i + 1], "%u", &parentPid) != 1) {
|
||||
fprintf(stderr, "couldn't set the parent PID to %s\n", argv[i + 1]);
|
||||
@@ -305,7 +293,7 @@ int main(int argc, char *argv[]) {
|
||||
}
|
||||
|
||||
/* launch */
|
||||
ret = help_server(argv[1], defaultPort, bindAddr);
|
||||
ret = help_server(argv[1], defaultPort);
|
||||
|
||||
htsthread_wait_n(background_threads - 1);
|
||||
hts_uninit();
|
||||
@@ -320,8 +308,10 @@ int main(int argc, char *argv[]) {
|
||||
static int webhttrack_runmain(httrackp * opt, int argc, char **argv);
|
||||
static void back_launch_cmd(void *pP) {
|
||||
char *cmd = (char *) pP;
|
||||
char **argv;
|
||||
char **argv = (char **) malloct(1024 * sizeof(char *));
|
||||
int argc = 0;
|
||||
int i = 0;
|
||||
int g = 0;
|
||||
|
||||
//
|
||||
httrackp *opt;
|
||||
@@ -332,19 +322,28 @@ static void back_launch_cmd(void *pP) {
|
||||
commandReturnCmdl = strdup(cmd);
|
||||
|
||||
/* split */
|
||||
argv = hts_split_cmdline(cmd, &argc);
|
||||
if (argv == NULL) {
|
||||
if (commandReturnMsg)
|
||||
free(commandReturnMsg);
|
||||
commandReturnMsg = strdup("could not parse the command line");
|
||||
commandReturn = -1;
|
||||
commandRunning = 0;
|
||||
commandEnd = 1;
|
||||
free(cmd);
|
||||
return;
|
||||
argv[0] = strdup("webhttrack");
|
||||
argv[1] = cmd;
|
||||
argc++;
|
||||
i = 0;
|
||||
while(cmd[i]) {
|
||||
if (cmd[i] == '\t' || cmd[i] == '\r' || cmd[i] == '\n') {
|
||||
cmd[i] = ' ';
|
||||
}
|
||||
i++;
|
||||
}
|
||||
i = 0;
|
||||
while(cmd[i]) {
|
||||
if (cmd[i] == '\"')
|
||||
g = !g;
|
||||
if (cmd[i] == ' ') {
|
||||
if (!g) {
|
||||
cmd[i] = '\0';
|
||||
argv[argc++] = cmd + i + 1;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
/* drop the program name the posted command line carries */
|
||||
argv[0] = strdupt("webhttrack");
|
||||
|
||||
/* init */
|
||||
hts_init();
|
||||
@@ -373,7 +372,6 @@ static void back_launch_cmd(void *pP) {
|
||||
|
||||
/* free */
|
||||
free(cmd);
|
||||
freet(argv[0]);
|
||||
freet(argv);
|
||||
return;
|
||||
}
|
||||
@@ -429,11 +427,11 @@ static int webhttrack_runmain(httrackp * opt, int argc, char **argv) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int help_server(char *dest_path, int defaultPort, const char *bindAddr) {
|
||||
static int help_server(char *dest_path, int defaultPort) {
|
||||
int returncode = 0;
|
||||
char adr_prox[HTS_URLMAXSIZE * 2];
|
||||
int port_prox;
|
||||
T_SOC soc = smallserver_init_std(&port_prox, adr_prox, defaultPort, bindAddr);
|
||||
T_SOC soc = smallserver_init_std(&port_prox, adr_prox, defaultPort);
|
||||
|
||||
if (soc != INVALID_SOCKET) {
|
||||
char url[HTS_URLMAXSIZE * 2];
|
||||
@@ -673,10 +671,10 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
|
||||
char *eps = strchr(back[i].url_adr, '/');
|
||||
int count;
|
||||
|
||||
if (ep != NULL && eps != NULL && ep < eps &&
|
||||
(count = (int) (ep - back[i].url_adr)) < 4) {
|
||||
if (ep != NULL && ep < eps
|
||||
&& (count = (int) (ep - back[i].url_adr)) < 4) {
|
||||
proto[0] = '\0';
|
||||
strncatbuff(proto, back[i].url_adr, count);
|
||||
strncat(proto, back[i].url_adr, count);
|
||||
}
|
||||
}
|
||||
snprintf(StatsBuffer[index].state, sizeof(StatsBuffer[index].state),
|
||||
|
||||
@@ -46,7 +46,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "httrack.h"
|
||||
#include "htslib.h"
|
||||
#include "htscharset.h" // after htslib.h: winsock2.h must precede windows.h
|
||||
#include "htsbacktrace.h"
|
||||
|
||||
/* Static definitions */
|
||||
static int fexist(const char *s);
|
||||
@@ -73,6 +72,10 @@ static int linput(FILE * fp, char *s, int max);
|
||||
#include <sys/ioctl.h>
|
||||
#endif
|
||||
#include <ctype.h>
|
||||
#if (defined(__linux) && defined(HAVE_EXECINFO_H))
|
||||
#include <execinfo.h>
|
||||
#define USES_BACKTRACE
|
||||
#endif
|
||||
/* END specific definitions */
|
||||
|
||||
static void __cdecl htsshow_init(t_hts_callbackarg * carg);
|
||||
@@ -877,6 +880,21 @@ static void sig_doback(int blind) { // mettre en backing
|
||||
#undef FD_ERR
|
||||
#define FD_ERR 2
|
||||
|
||||
static void print_backtrace(void) {
|
||||
#ifdef USES_BACKTRACE
|
||||
void *stack[256];
|
||||
const int size = backtrace(stack, sizeof(stack)/sizeof(stack[0]));
|
||||
if (size != 0) {
|
||||
backtrace_symbols_fd(stack, size, FD_ERR);
|
||||
}
|
||||
#else
|
||||
const char msg[] = "No stack trace available on this OS :(\n";
|
||||
if (write(FD_ERR, msg, sizeof(msg) - 1) != sizeof(msg) - 1) {
|
||||
/* sorry GCC */
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static size_t print_num(char *buffer, int num) {
|
||||
size_t i, j;
|
||||
if (num < 0) {
|
||||
@@ -910,7 +928,7 @@ static void sig_fatal(int code) {
|
||||
size += print_num(&buffer[size], code);
|
||||
buffer[size++] = '\n';
|
||||
(void) (write(FD_ERR, buffer, size) == size);
|
||||
hts_print_backtrace(FD_ERR);
|
||||
print_backtrace();
|
||||
(void) (write(FD_ERR, msgreport, sizeof(msgreport) - 1)
|
||||
== sizeof(msgreport) - 1);
|
||||
abort();
|
||||
@@ -933,7 +951,6 @@ static void sig_leave(int code) {
|
||||
}
|
||||
|
||||
static void signal_handlers(void) {
|
||||
hts_backtrace_init();
|
||||
#ifdef _WIN32
|
||||
signal(SIGINT, sig_leave); // ^C
|
||||
signal(SIGTERM, sig_finish); // kill <process>
|
||||
|
||||
@@ -100,7 +100,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ClCompile Include="httrack.c" />
|
||||
<ClCompile Include="htsbacktrace.c" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Pulls in libhttrack.lib and builds it first. -->
|
||||
|
||||
@@ -59,10 +59,9 @@
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<!-- LIBHTTRACK_EXPORTS turns HTSEXT_API into __declspec(dllexport); ZLIB_DLL
|
||||
imports zlib from its DLL, ZLIB_CONST makes its next_in const as the
|
||||
autotools build does. HTS_USEBROTLI/HTS_USEZSTD enable the br and zstd
|
||||
content codings. Windows 7 floor, matching WinHTTrack. -->
|
||||
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;ZLIB_CONST;HTS_USEBROTLI=1;HTS_USEZSTD=1;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
imports zlib from its DLL. HTS_USEBROTLI/HTS_USEZSTD enable the br and
|
||||
zstd content codings. Windows 7 floor, matching WinHTTrack. -->
|
||||
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;HTS_USEBROTLI=1;HTS_USEZSTD=1;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>$(MSBuildThisFileDirectory);$(MSBuildThisFileDirectory)coucal;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
@@ -122,7 +121,6 @@
|
||||
<ClCompile Include="htshelp.c" />
|
||||
<ClCompile Include="htsindex.c" />
|
||||
<ClCompile Include="htslib.c" />
|
||||
<ClCompile Include="htscmdline.c" />
|
||||
<ClCompile Include="htsurlport.c" />
|
||||
<ClCompile Include="htsmd5.c" />
|
||||
<ClCompile Include="htsmodules.c" />
|
||||
@@ -131,7 +129,6 @@
|
||||
<ClCompile Include="htsproxy.c" />
|
||||
<ClCompile Include="htsrobots.c" />
|
||||
<ClCompile Include="htsselftest.c" />
|
||||
<ClCompile Include="htssinglefile.c" />
|
||||
<ClCompile Include="htssniff.c" />
|
||||
<ClCompile Include="htsthread.c" />
|
||||
<ClCompile Include="htstools.c" />
|
||||
|
||||
@@ -771,7 +771,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
|
||||
PT_ReadIndex(indexes, StringBuff(itemUrl) + 1, FETCH_HEADERS);
|
||||
if (file != NULL && file->statuscode == HTTP_OK) {
|
||||
size = file->size;
|
||||
if (file->lastmodified[0] != '\0') {
|
||||
if (file->lastmodified) {
|
||||
timestamp = get_time_rfc822(file->lastmodified);
|
||||
}
|
||||
if (timestamp == (time_t) 0) {
|
||||
@@ -785,7 +785,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
|
||||
}
|
||||
timestamp = timestampRep;
|
||||
}
|
||||
if (file->contenttype[0] != '\0') {
|
||||
if (file->contenttype) {
|
||||
mimeType = file->contenttype;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -579,10 +579,9 @@ PT_Index PT_LoadCache(const char *filename) {
|
||||
if (chain != NULL) {
|
||||
const char *scheme = link_has_authority(chain->name) ? "" : "http://";
|
||||
|
||||
/* dropped rather than truncated: empty already reads as "unset" */
|
||||
if (!sprintfbuff(index->slots.common.startUrl, "%s%s", scheme,
|
||||
(const char *) chain->name))
|
||||
index->slots.common.startUrl[0] = '\0';
|
||||
snprintf(index->slots.common.startUrl,
|
||||
sizeof(index->slots.common.startUrl), "%s%s", scheme,
|
||||
(const char *) chain->name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -973,12 +972,9 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
|
||||
const char *scheme =
|
||||
link_has_authority(filenameIndex) ? "" : "http://";
|
||||
|
||||
/* dropped rather than truncated; try the next entry */
|
||||
if (sprintfbuff(index->startUrl, "%s%s", scheme,
|
||||
filenameIndex))
|
||||
firstSeen = 1;
|
||||
else
|
||||
index->startUrl[0] = '\0';
|
||||
firstSeen = 1;
|
||||
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
|
||||
scheme, filenameIndex);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1574,11 +1570,9 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
const char *scheme =
|
||||
link_has_authority(line) ? "" : "http://";
|
||||
|
||||
/* dropped rather than truncated; try the next entry */
|
||||
if (sprintfbuff(index->startUrl, "%s%s", scheme, line))
|
||||
firstSeen = 1;
|
||||
else
|
||||
index->startUrl[0] = '\0';
|
||||
firstSeen = 1;
|
||||
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
|
||||
scheme, line);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2197,7 +2191,7 @@ static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
|
||||
}
|
||||
if ((pos = getArcField(index->line, 2)) != NULL) {
|
||||
r->msg[0] = '\0';
|
||||
strncatbuff(r->msg, pos, sizeof(r->msg) - 1);
|
||||
strncat(r->msg, pos, sizeof(pos) - 1);
|
||||
}
|
||||
while(linput(index->file, index->line, sizeof(index->line) - 1)
|
||||
&& index->line[0] != '\0') {
|
||||
|
||||
@@ -97,7 +97,6 @@
|
||||
<ItemGroup>
|
||||
<ClCompile Include="htsserver.c" />
|
||||
<ClCompile Include="htsweb.c" />
|
||||
<ClCompile Include="htscmdline.c" />
|
||||
<ClCompile Include="htsurlport.c" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# webhttrack posts its command line as one string: the argv split must grow past
|
||||
# 1024 arguments and keep a quote inside a value out of the option parser.
|
||||
httrack -O /dev/null -#test=cmdline-split run | grep -q "cmdline-split self-test OK"
|
||||
@@ -1,25 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Buffer capacity for a 64-bit file size ('httrack -#test=growsize'): a -%S list
|
||||
# file past 4GB must size its buffer exactly or be refused, never wrap.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
out=$(httrack -#test=growsize)
|
||||
echo "$out"
|
||||
test "$out" == "growsize self-test OK"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_growsize.XXXXXX")
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
echo '<html><body>hi</body></html>' >"$tmp/index.html"
|
||||
printf -- '-*/zzmarker*\n' >"$tmp/rules.txt"
|
||||
|
||||
# the rules file lands in the URL/filter string, echoed back by the banner
|
||||
run=$(httrack -O "$tmp/out" --quiet -n "-%S" "$tmp/rules.txt" \
|
||||
"file://$tmp/index.html" 2>&1) || true
|
||||
printf '%s\n' "$run" | grep -q 'zzmarker' || {
|
||||
echo "FAIL: -%S rules file was not loaded"
|
||||
printf '%s\n' "$run"
|
||||
exit 1
|
||||
}
|
||||
@@ -47,18 +47,3 @@ case "$err" in
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# An array source with no NUL must abort on the source bound rather than run
|
||||
# off the array: the array half of the source-capacity selection.
|
||||
err=$(httrack -#test=strsafe overflow-src "x" 2>&1) || true
|
||||
case "$err" in
|
||||
*"strsafe: NOT aborted"*)
|
||||
echo "unterminated source array was NOT caught" >&2
|
||||
exit 1
|
||||
;;
|
||||
*"size < sizeof_source"*) ;;
|
||||
*)
|
||||
echo "expected htssafe source-bound abort, got: $err" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/bin/bash
|
||||
# An ARC entry's HTTP reason phrase must survive a proxytrack --convert
|
||||
# round-trip; it used to be clipped to sizeof(char*) - 1 bytes.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
printf 'HTTP/1.1 404 Not Found Here At All\r\nContent-Type: text/html\r\nLast-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
|
||||
printf 'hello' >"$dir/body"
|
||||
alen=$(($(wc -c <"$dir/hdr") + $(wc -c <"$dir/body")))
|
||||
|
||||
# ARC 1.0: filedesc record and version block, then per entry
|
||||
# <nl> <URL-record> <nl> <headers> <body>; the record's last field is that length.
|
||||
{
|
||||
printf 'filedesc://t.arc 0.0.0.0 20250101000000 text/plain 200 - - 0 t.arc 9\n'
|
||||
printf '2 0 test\n'
|
||||
printf '\n\n'
|
||||
printf 'http://example.com/page.html 0.0.0.0 20250101000000 text/html 404 - - 0 t.arc %d\n' "$alen"
|
||||
cat "$dir/hdr" "$dir/body"
|
||||
} >"$dir/in.arc"
|
||||
|
||||
proxytrack --convert "$dir/out.arc" "$dir/in.arc" >/dev/null 2>&1
|
||||
|
||||
# HTTP/1.0 is the writer's own prefix (the input says 1.1), so a verbatim copy
|
||||
# of the input cannot satisfy this match.
|
||||
grep -aqF 'HTTP/1.0 404 Not Found Here At All' "$dir/out.arc" || {
|
||||
echo "reason phrase lost in the ARC round-trip:" >&2
|
||||
grep -a '^HTTP/' "$dir/out.arc" >&2 || echo "(no status line at all)" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -60,20 +60,13 @@ test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
|
||||
|
||||
# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset.
|
||||
"${python}" - "${url}" "${work}" <<'PY'
|
||||
import re, sys, urllib.parse, urllib.request
|
||||
import sys, urllib.parse, urllib.request
|
||||
|
||||
url, work = sys.argv[1], sys.argv[2]
|
||||
outdir = work + "/caf\xe9" # 'café' as the single byte the browser would send
|
||||
# Port 1 refuses at once: only the decoded -O dir matters, not the fetch.
|
||||
cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir
|
||||
# The body is refused without the session id the server renders into each form.
|
||||
# Note the wizard page is under /server/; a bare /step4.html is the doc page.
|
||||
page = urllib.request.urlopen(url + "server/step4.html", timeout=20).read()
|
||||
m = re.search(rb'name="sid" value="([0-9a-f]+)"', page)
|
||||
if not m:
|
||||
raise SystemExit("no session id in server/step4.html")
|
||||
fields = [("sid", m.group(1).decode()),
|
||||
("path", work), ("projname", "proj"), ("command_do", "start"),
|
||||
fields = [("path", work), ("projname", "proj"), ("command_do", "start"),
|
||||
("winprofile", "x"), ("command", cmd)]
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1"))
|
||||
for k, v in fields)
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# htsserver's POST redirect: the Location header is built from a client-supplied
|
||||
# value, and the listen address is loopback unless --bind widens it.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
# run_with_timeout/kill_tree: timeout(1) is absent on macOS
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
command -v python3 >/dev/null || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
log=$(mktemp)
|
||||
# start() runs inside a command substitution, so its $! never reaches this
|
||||
# shell. Take the pid from the server's own announcement instead: a missed kill
|
||||
# leaves an orphan holding the CI job open long after the suite has passed.
|
||||
srvpid() { sed -n 's/^PID=//p' "${log}" 2>/dev/null | head -1; }
|
||||
cleanup() {
|
||||
stop
|
||||
rm -f "${log}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()'
|
||||
}
|
||||
|
||||
# Start htsserver, echo the announced URL. Extra args are passed through.
|
||||
start() {
|
||||
local port url
|
||||
port=$(freeport)
|
||||
: >"${log}"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
exec htsserver "${distdir}/" --port "${port}" "$@" >"${log}" 2>&1
|
||||
) &
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
|
||||
echo "${url}"
|
||||
}
|
||||
|
||||
stop() {
|
||||
local pid
|
||||
pid=$(srvpid)
|
||||
test -z "${pid}" || kill -9 "${pid}" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# The session id gates the request body, so a POST has to carry one. The server
|
||||
# renders it into every form, which is where a browser picks it up too.
|
||||
scrape_sid() {
|
||||
python3 -c 'import socket, sys
|
||||
s = socket.create_connection(("127.0.0.1", int(sys.argv[1])), 10)
|
||||
s.settimeout(20)
|
||||
s.sendall(b"GET /server/index.html HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n")
|
||||
out = b""
|
||||
while True:
|
||||
b = s.recv(65536)
|
||||
if not b:
|
||||
break
|
||||
out += b
|
||||
s.close()
|
||||
sys.stdout.write(out.decode("latin-1"))' "$1" |
|
||||
sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1
|
||||
}
|
||||
|
||||
# POST redirect=$1 to 127.0.0.1:$2 with session id $3, print the raw headers.
|
||||
post_redirect() {
|
||||
python3 -c 'import socket, sys, urllib.parse
|
||||
body = ("sid=" + sys.argv[3] + "&redirect="
|
||||
+ urllib.parse.quote(sys.argv[1], safe=""))
|
||||
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
|
||||
"Content-type: application/x-www-form-urlencoded\r\n"
|
||||
"Content-length: %d\r\n\r\n%s" % (len(body), body))
|
||||
s = socket.create_connection(("127.0.0.1", int(sys.argv[2])), 10)
|
||||
s.settimeout(20)
|
||||
s.sendall(req.encode())
|
||||
out = b""
|
||||
# stop at the end of the header block: the server need not close the connection,
|
||||
# and an unbounded recv() hung the macOS runner for over an hour.
|
||||
while b"\r\n\r\n" not in out:
|
||||
b = s.recv(65536)
|
||||
if not b:
|
||||
break
|
||||
out += b
|
||||
s.close()
|
||||
sys.stdout.write(out.split(b"\r\n\r\n")[0].decode("latin-1"))' "$1" "$2" "$3"
|
||||
}
|
||||
|
||||
portof() { echo "${1##*:}" | tr -d /; }
|
||||
|
||||
# "free"/"inuse"/"unusable": can $1 still be bound on 127.0.0.2? A wildcard
|
||||
# listener takes the port on every address, a loopback-only one does not, so
|
||||
# this discriminates where the announced URL cannot.
|
||||
probe_alias() {
|
||||
python3 -c 'import socket, sys
|
||||
s = socket.socket()
|
||||
try:
|
||||
s.bind(("127.0.0.2", int(sys.argv[1])))
|
||||
except OSError as e:
|
||||
print("unusable" if e.errno in (99, 49, 10049) else "inuse")
|
||||
else:
|
||||
print("free")
|
||||
finally:
|
||||
s.close()' "$1"
|
||||
}
|
||||
|
||||
# Non-repeating and not a round number: a truncation, a reorder or a dropped
|
||||
# interior byte all stay visible.
|
||||
long=$(python3 -c 'print("".join(chr(33 + i % 90) for i in range(4097)))')
|
||||
url=$(start)
|
||||
port=$(portof "${url}")
|
||||
sid=$(scrape_sid "${port}")
|
||||
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid (got '${sid}')"
|
||||
resp=$(post_redirect "${long}" "${port}" "${sid}") ||
|
||||
fail "no response to an oversized redirect value"
|
||||
stop
|
||||
loc=$(printf '%s' "${resp}" | sed -n 's/^Location: //p' | tr -d '\r')
|
||||
test "${loc}" = "${long}" ||
|
||||
fail "oversized redirect not echoed whole (got ${#loc} bytes, want ${#long})"
|
||||
|
||||
# CR/LF must suppress the header outright. Sanitising it instead would keep the
|
||||
# grep for an injected header quiet while still emitting a mangled Location.
|
||||
url=$(start)
|
||||
port=$(portof "${url}")
|
||||
sid=$(scrape_sid "${port}")
|
||||
resp=$(post_redirect '/foo
|
||||
X-Injected: pwned' "${port}" "${sid}") || fail "no response to a CRLF redirect value"
|
||||
stop
|
||||
printf '%s' "${resp}" | grep -qi 'X-Injected' &&
|
||||
fail "CRLF in the redirect value reached the response"
|
||||
test "$(printf '%s' "${resp}" | grep -ci '^Location:')" -eq 0 ||
|
||||
fail "CRLF redirect still emitted a Location header"
|
||||
test "$(printf '%s' "${resp}" | grep -c '^HTTP/1\.')" -eq 1 ||
|
||||
fail "CRLF redirect did not yield exactly one status line"
|
||||
|
||||
# Loopback unless asked otherwise. Assert the socket, not the announcement.
|
||||
url=$(start)
|
||||
port=$(portof "${url}")
|
||||
alias_state=$(probe_alias "${port}")
|
||||
stop
|
||||
if test "${alias_state}" = unusable; then
|
||||
echo "no 127.0.0.2 alias; skipping the listen-address assertions" >&2
|
||||
else
|
||||
test "${alias_state}" = free ||
|
||||
fail "default listen address is not loopback-only (127.0.0.2:${port} taken)"
|
||||
case "${url}" in
|
||||
http://127.0.0.1:*) ;;
|
||||
*) fail "default announcement is not loopback: ${url}" ;;
|
||||
esac
|
||||
|
||||
url=$(start --bind 0.0.0.0)
|
||||
port=$(portof "${url}")
|
||||
alias_state=$(probe_alias "${port}")
|
||||
stop
|
||||
test "${alias_state}" = inuse ||
|
||||
fail "--bind 0.0.0.0 did not take the wildcard (127.0.0.2:${port} ${alias_state})"
|
||||
fi
|
||||
|
||||
# An empty --bind must not quietly fall back to every interface. Bounded: if it
|
||||
# were accepted the server would listen instead of exiting.
|
||||
run_with_timeout 15 htsserver "${distdir}/" --bind "" >"${log}" 2>&1 || true
|
||||
grep -q -- "--bind needs an address" "${log}" ||
|
||||
fail "empty --bind not refused: $(cat "${log}")"
|
||||
|
||||
echo "PASS"
|
||||
@@ -1,142 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# htsserver's session id must gate the request body: every field in it is
|
||||
# written into the global key store, and "command" from there reaches the engine.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
command -v python3 >/dev/null || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
srv=
|
||||
log=$(mktemp)
|
||||
cleanup() {
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
rm -f "${log}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()'
|
||||
}
|
||||
|
||||
# Echo the announced URL. Runs in a command substitution, so it is a
|
||||
# subshell and cannot export the pid: the caller reads it back with srvpid.
|
||||
start() {
|
||||
local port url
|
||||
port=$(freeport)
|
||||
: >"${log}"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
|
||||
) &
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
|
||||
echo "${url}"
|
||||
}
|
||||
|
||||
# The server reports its own pid; the aliveness assertions below hang off it.
|
||||
srvpid() { sed -n 's/^PID=//p' "${log}" | head -1; }
|
||||
|
||||
alive() { kill -0 "$1" 2>/dev/null; }
|
||||
|
||||
portof() { echo "${1##*:}" | tr -d /; }
|
||||
|
||||
# Raw request to 127.0.0.1:$1; $2 is the body ("" for a GET of the $3 page,
|
||||
# default index). Prints the reply.
|
||||
request() {
|
||||
python3 -c 'import socket, sys
|
||||
port, body = int(sys.argv[1]), sys.argv[2]
|
||||
page = sys.argv[3] if len(sys.argv) > 3 else "index"
|
||||
if body:
|
||||
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
|
||||
"Content-type: application/x-www-form-urlencoded\r\n"
|
||||
"Content-length: %d\r\n\r\n%s" % (len(body), body))
|
||||
else:
|
||||
req = ("GET /server/%s.html HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % page)
|
||||
s = socket.create_connection(("127.0.0.1", port), 10)
|
||||
s.settimeout(30)
|
||||
s.sendall(req.encode())
|
||||
out = b""
|
||||
while True:
|
||||
b = s.recv(65536)
|
||||
if not b:
|
||||
break
|
||||
out += b
|
||||
s.close()
|
||||
sys.stdout.write(out.decode("latin-1"))' "$1" "$2" ${3+"$3"}
|
||||
}
|
||||
|
||||
# The server hands the sid to any client in every form; scraping it is the
|
||||
# legitimate flow, and it is what makes the accept case below meaningful.
|
||||
scrape_sid() {
|
||||
request "$1" "" | sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1
|
||||
}
|
||||
|
||||
url=$(start)
|
||||
port=$(portof "${url}")
|
||||
srv=$(srvpid)
|
||||
test -n "${srv}" || fail "htsserver did not report its pid"
|
||||
alive "${srv}" || fail "htsserver is not running"
|
||||
|
||||
sid=$(scrape_sid "${port}")
|
||||
# Control: without a real sid every assertion below would pass vacuously.
|
||||
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid from the page (got '${sid}')"
|
||||
|
||||
# Accept: the legitimate flow still works. "redirect" is the cheapest field
|
||||
# with a reply that is visible in the headers.
|
||||
resp=$(request "${port}" "sid=${sid}&redirect=/accepted")
|
||||
printf '%s' "${resp}" | grep -q '^Location: /accepted' ||
|
||||
fail "a body carrying the correct sid was refused"
|
||||
|
||||
# Refuse: missing and wrong. A missing one is the regression under test — the
|
||||
# expected value used to be pre-seeded into the compared key, so omitting the
|
||||
# field compared equal to itself.
|
||||
for bad in "redirect=/nosid" "sid=&redirect=/empty" \
|
||||
"sid=00000000000000000000000000000000&redirect=/wrong"; do
|
||||
resp=$(request "${port}" "${bad}")
|
||||
printf '%s' "${resp}" | grep -q '^Location:' &&
|
||||
fail "body accepted without a valid sid: ${bad}"
|
||||
# A refusal has to be a well-formed reply, not a headerless fragment: the
|
||||
# release build used to emit only Content-length, which reads as a protocol
|
||||
# error to any client and hides the reason.
|
||||
printf '%s' "${resp}" | grep -q '^HTTP/1\.0 403 ' ||
|
||||
fail "refusal was not a 403: ${bad}"
|
||||
done
|
||||
|
||||
# Suppressing the reply is not the same as refusing the write: the body is
|
||||
# applied to one global key store that later requests render from, and the
|
||||
# command dispatcher reads it from there. Probe the store itself — step3
|
||||
# interpolates ${projname} into its title — rather than the refused reply.
|
||||
title() { request "$1" "" step3; }
|
||||
|
||||
request "${port}" "projname=UNAUTHWRITE" >/dev/null 2>&1 || true
|
||||
title "${port}" | grep -q 'UNAUTHWRITE' &&
|
||||
fail "a body without a valid sid was written to the key store"
|
||||
|
||||
# Paired accept case: without it the assertion above passes even if the server
|
||||
# simply ignores every body, which would prove nothing.
|
||||
request "${port}" "sid=${sid}&projname=AUTHWRITE" >/dev/null 2>&1 || true
|
||||
title "${port}" | grep -q 'AUTHWRITE' ||
|
||||
fail "a body carrying the correct sid was not written to the key store"
|
||||
|
||||
echo "PASS"
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# proxytrack's WebDAV PROPFIND fallback must behave identically whether or not
|
||||
# the cache entry supplies Content-Type and Last-Modified.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
python=$(find_python) || {
|
||||
echo "python3 missing, skipping"
|
||||
exit 77
|
||||
}
|
||||
command -v curl >/dev/null 2>&1 || {
|
||||
echo "curl missing, skipping"
|
||||
exit 77
|
||||
}
|
||||
# First test to run proxytrack as a live server, and MSYS cannot reap a native
|
||||
# listener: the orphan wedged the whole Windows suite for 48 minutes (#595).
|
||||
if is_windows; then
|
||||
echo "windows: cannot reap a backgrounded proxytrack, skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
dir=$(mktemp -d)
|
||||
ptpid=
|
||||
cleanup() {
|
||||
stop_server "$ptpid"
|
||||
rm -rf "$dir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Neither header is present, so file->contenttype and ->lastmodified stay "".
|
||||
printf 'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
|
||||
printf 'hello' >"$dir/body"
|
||||
alen=$(($(wc -c <"$dir/hdr") + $(wc -c <"$dir/body")))
|
||||
{
|
||||
printf 'filedesc://t.arc 0.0.0.0 20250101000000 text/plain 200 - - 0 t.arc 9\n'
|
||||
printf '2 0 test\n'
|
||||
printf '\n\n'
|
||||
printf 'http://example.com/page.html 0.0.0.0 20250101000000 text/html 200 - - 0 t.arc %d\n' "$alen"
|
||||
cat "$dir/hdr" "$dir/body"
|
||||
} >"$dir/in.arc"
|
||||
|
||||
freeport() {
|
||||
"$python" -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()'
|
||||
}
|
||||
proxyport=$(freeport)
|
||||
icpport=$(freeport)
|
||||
|
||||
proxytrack "127.0.0.1:$proxyport" "127.0.0.1:$icpport" "$dir/in.arc" >"$dir/pt.log" 2>&1 &
|
||||
ptpid=$!
|
||||
waited=0
|
||||
until grep -qE "HTTP Proxy installed on|Unable to (initialize a temporary server|create the server)" "$dir/pt.log"; do
|
||||
kill -0 "$ptpid" 2>/dev/null || {
|
||||
echo "FAIL: proxytrack exited before listening"
|
||||
cat "$dir/pt.log"
|
||||
exit 1
|
||||
}
|
||||
test "$waited" -lt 50 || {
|
||||
echo "FAIL: proxytrack never announced its listen port"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
grep -q "HTTP Proxy installed on" "$dir/pt.log" || {
|
||||
echo "FAIL: proxytrack failed to bind"
|
||||
cat "$dir/pt.log"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --max-time: an unbounded read here would wedge the runner rather than fail
|
||||
curl -s --max-time 30 -X PROPFIND -H "Depth: 1" \
|
||||
"http://127.0.0.1:$proxyport/webdav/example.com/" >"$dir/resp.xml"
|
||||
|
||||
grep -q '<getcontenttype>application/octet-stream</getcontenttype>' "$dir/resp.xml" || {
|
||||
echo "FAIL: missing Content-Type did not fall back to application/octet-stream"
|
||||
cat "$dir/resp.xml"
|
||||
exit 1
|
||||
}
|
||||
grep -q '<getlastmodified>Wed, 01 Jan 2025 00:00:00 GMT</getlastmodified>' "$dir/resp.xml" || {
|
||||
echo "FAIL: missing Last-Modified did not fall back to the index timestamp"
|
||||
cat "$dir/resp.xml"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "OK: WebDAV listing falls back correctly for an entry with no Content-Type/Last-Modified"
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/bin/bash
|
||||
# A fatal signal prints the raw backtrace, then names the frames
|
||||
# backtrace_symbols_fd() leaves as module+offset (-fvisibility=hidden hides them).
|
||||
set -eu
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
if is_windows; then
|
||||
echo "no backtrace() on Windows; skipping" >&2
|
||||
exit 77
|
||||
fi
|
||||
command -v httrack >/dev/null || {
|
||||
echo "could not find httrack" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_symbolize.XXXXXX") || exit 1
|
||||
trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
|
||||
out="${tmpdir}/trace"
|
||||
raw="${tmpdir}/trace-optout"
|
||||
overflow="this string is far too long for the buffer"
|
||||
# A frame glibc could not name. Stop at the offset: it prints the trailing
|
||||
# "[0xADDR]" with or without a leading space depending on its version.
|
||||
rawframe='(+0x[0-9a-f][0-9a-f]*)'
|
||||
|
||||
# The strsafe selftest aborts on purpose, which lands in sig_fatal.
|
||||
rc=0
|
||||
run_with_timeout 60 httrack -#test=strsafe overflow "$overflow" >"$out" 2>&1 || rc=$?
|
||||
test "$rc" -ne 124 || {
|
||||
echo "the crash handler did not finish within the deadline" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
grep -q '^Caught signal ' "$out" || {
|
||||
echo "no 'Caught signal' line:" >&2
|
||||
cat "$out" >&2
|
||||
exit 1
|
||||
}
|
||||
if grep -q 'No stack trace available on this OS' "$out"; then
|
||||
echo "no backtrace() on this platform; skipping" >&2
|
||||
exit 77
|
||||
fi
|
||||
# Unconditional and first: symbolizing must never cost the raw trace.
|
||||
grep -q "$rawframe" "$out" || {
|
||||
echo "raw module+offset frames are gone:" >&2
|
||||
cat "$out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v addr2line >/dev/null || {
|
||||
echo "addr2line not installed; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
# Control: resolve the same frames ourselves, so a stripped build cannot turn
|
||||
# the assertion below into a vacuous pass.
|
||||
oracle="${tmpdir}/oracle"
|
||||
sed -n 's/^\([^(]*\)(+\(0x[0-9a-f]*\)).*$/\1 \2/p' "$out" |
|
||||
while read -r mod off; do
|
||||
test -f "$mod" || continue
|
||||
addr2line -Cfip -e "$mod" "$off" 2>/dev/null || true
|
||||
done >"$oracle"
|
||||
grep -qE 'st_strsafe|strcpy_safe_' "$oracle" || {
|
||||
echo "addr2line names no hidden symbol in this build; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
# The payload. Dropping the raw frames first is what makes it specific: both
|
||||
# names are static, so .dynsym could never have carried them. Not anchored on
|
||||
# the "0xOFF:" line, the inline chain puts the name on either half.
|
||||
grep -v "$rawframe" "$out" | grep -qE 'st_strsafe|strcpy_safe_' || {
|
||||
echo "the handler did not name the hidden frames:" >&2
|
||||
cat "$out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Opt-out: raw trace kept, nothing spawned.
|
||||
export HTTRACK_NO_SYMBOLIZE=1
|
||||
rc=0
|
||||
run_with_timeout 60 httrack -#test=strsafe overflow "$overflow" >"$raw" 2>&1 || rc=$?
|
||||
unset HTTRACK_NO_SYMBOLIZE
|
||||
test "$rc" -ne 124 || {
|
||||
echo "the opt-out run did not finish within the deadline" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "$rawframe" "$raw" || {
|
||||
echo "the opt-out lost the raw trace:" >&2
|
||||
cat "$raw" >&2
|
||||
exit 1
|
||||
}
|
||||
! grep -q '^0x[0-9a-f]*: ' "$raw" || {
|
||||
echo "the opt-out still symbolized:" >&2
|
||||
cat "$raw" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The wizard's size fields must render as distinct caps: sizemax is -M, othermax
|
||||
# then maxhtml share -m.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
python=$(find_python) || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_maxsize.XXXXXX") || fail "no tmpdir"
|
||||
srvlog=$(mktemp)
|
||||
srv=
|
||||
cleanup() {
|
||||
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
srv=
|
||||
rm -rf "${work}" "${srvlog}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
|
||||
# ~/.httrack.ini out of it.
|
||||
sport=$("${python}" -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()')
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
export HOME="${work}"
|
||||
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
|
||||
) &
|
||||
srv=$!
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
|
||||
kill -0 "${srv}" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
|
||||
|
||||
# Audit what step4.html renders back; without command_do nothing is launched.
|
||||
"${python}" - "${url}" <<'PY' || fail "wizard rendered the wrong options (see above)"
|
||||
import re, sys, urllib.parse, urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
html, other, site = "111111", "222222", "333333"
|
||||
|
||||
|
||||
def post(fields):
|
||||
# The body is refused without the session id the server puts in each form.
|
||||
form = urllib.request.urlopen(url + "server/index.html", timeout=20).read()
|
||||
m = re.search(rb'name="sid" value="([0-9a-f]+)"', form)
|
||||
if not m:
|
||||
raise SystemExit("no session id in server/index.html")
|
||||
fields = [("sid", m.group(1).decode())] + fields
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v)) for k, v in fields)
|
||||
req = urllib.request.Request(url + "server/step4.html",
|
||||
data=body.encode("latin-1"), method="POST")
|
||||
return urllib.request.urlopen(req, timeout=20).read().decode("latin-1")
|
||||
|
||||
|
||||
def textarea(page, name):
|
||||
m = re.search(r'<textarea name="%s".*?>(.*?)</textarea>' % name, page, re.S)
|
||||
if m is None:
|
||||
sys.exit("no %s textarea in the rendered step4.html" % name)
|
||||
return m.group(1)
|
||||
|
||||
|
||||
def want(ok, msg, ctx):
|
||||
if not ok:
|
||||
sys.exit("%s\nrendered:%s" % (msg, ctx))
|
||||
|
||||
|
||||
page = post([("maxhtml", html), ("othermax", other), ("sizemax", site),
|
||||
("dos", "2")])
|
||||
cmd = textarea(page, "command")
|
||||
want("--max-size=" + site in cmd, "site cap not emitted as --max-size", cmd)
|
||||
want("--max-files=" + site not in cmd, "site cap still an --max-files", cmd)
|
||||
# Positive controls: both per-file caps still ride -m, one option each.
|
||||
want("--max-files=" + other in cmd, "non-HTML cap not emitted as --max-files",
|
||||
cmd)
|
||||
want("--max-files=," + html in cmd, "HTML cap not emitted as --max-files=,",
|
||||
cmd)
|
||||
want(cmd.count("--max-files=") == 2, "unexpected --max-files count", cmd)
|
||||
want(cmd.index("--max-files=" + other) < cmd.index("--max-files=," + html),
|
||||
"the bare --max-files must precede the --max-files=, form", cmd)
|
||||
|
||||
# winprofile.ini feeds the same three caps to the Windows GUI, one key each.
|
||||
ini = textarea(page, "winprofile").replace("\r\n", "\n") # the ini is CRLF
|
||||
want("\nDos=2" in ini, "Dos not written from the dos field", ini)
|
||||
want("\nMaxHtml=" + html + "\n" in ini, "MaxHtml not written from maxhtml", ini)
|
||||
want("\nMaxOther=" + other + "\n" in ini, "MaxOther not written from othermax",
|
||||
ini)
|
||||
want("\nMaxAll=" + site + "\n" in ini, "MaxAll not written from sizemax", ini)
|
||||
|
||||
# A bare --max-files= (an unguarded empty field) parses as -m with no digits,
|
||||
# silently clearing the html cap.
|
||||
page = post([("maxhtml", ""), ("othermax", ""), ("sizemax", ""), ("dos", "2")])
|
||||
cmd = textarea(page, "command")
|
||||
want("--max-rate=" in cmd, "empty size fields rendered no command line", cmd)
|
||||
want("--max-files" not in cmd and "--max-size" not in cmd,
|
||||
"empty size fields still rendered a size option", cmd)
|
||||
|
||||
# A mistyped ${LANG_OK] key renders the OK button's label empty.
|
||||
opts = urllib.request.urlopen(url + "server/option2b.html",
|
||||
timeout=20).read().decode("latin-1")
|
||||
want("LANG_OK" not in opts, "unexpanded LANG_OK in option2b.html", "")
|
||||
want('<input type="submit" value="OK"' in opts, "no OK label in option2b.html",
|
||||
"")
|
||||
PY
|
||||
|
||||
cleanup
|
||||
|
||||
# A bare -m<n> resets the html limit, so only the bare-then-comma order keeps
|
||||
# both caps live. basic.html is 487 bytes, well past the 10-byte html cap.
|
||||
crawl() {
|
||||
bash "${distdir}/tests/local-crawl.sh" "$@"
|
||||
}
|
||||
crawl --errors 1 --log-found 'File too big' \
|
||||
httrack 'BASEURL/simple/basic.html' '--max-files=,10'
|
||||
crawl --errors 1 --log-found 'File too big' \
|
||||
httrack 'BASEURL/simple/basic.html' '--max-files=500000' '--max-files=,10'
|
||||
crawl --errors 0 --found 'simple/basic.html' --log-not-found 'File too big' \
|
||||
httrack 'BASEURL/simple/basic.html' '--max-files=,10' '--max-files=500000'
|
||||
|
||||
echo "PASS"
|
||||
@@ -1,303 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Issue #713: --single-file rewrites the mirrored pages so their assets ride
|
||||
# along as data: URIs. End to end over a real crawl, with the payloads decoded
|
||||
# and compared against what the server actually served.
|
||||
set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_713.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
htssrv=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
|
||||
test -z "$htssrv" || kill -9 "$htssrv" 2>/dev/null || true
|
||||
wait "$htssrv" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
htssrv=
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# --- the site ---------------------------------------------------------------
|
||||
doc="${tmpdir}/doc"
|
||||
mkdir -p "$doc"
|
||||
cat >"${doc}/index.html" <<'EOF'
|
||||
<html><head>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<link rel="canonical" href="other.html">
|
||||
</head><body>
|
||||
<img src="pixel.svg" alt="p">
|
||||
<img src="big.svg" alt="b">
|
||||
<script src="app.js"></script>
|
||||
<a href="other.html">other</a>
|
||||
</body></html>
|
||||
EOF
|
||||
cat >"${doc}/other.html" <<'EOF'
|
||||
<html><body><a href="index.html">back</a></body></html>
|
||||
EOF
|
||||
cat >"${doc}/style.css" <<'EOF'
|
||||
@import "nested.css";
|
||||
body { background: url(pixel.svg); }
|
||||
EOF
|
||||
cat >"${doc}/nested.css" <<'EOF'
|
||||
div { background: url(pixel.svg); }
|
||||
EOF
|
||||
printf 'var app = 1;\n' >"${doc}/app.js"
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><rect width="1" height="1"/></svg>\n' \
|
||||
>"${doc}/pixel.svg"
|
||||
{
|
||||
printf '<svg xmlns="http://www.w3.org/2000/svg"><!--'
|
||||
head -c 20000 /dev/zero | tr '\0' 'x'
|
||||
printf '%s\n' '--><rect width="1" height="1"/></svg>'
|
||||
} >"${doc}/big.svg"
|
||||
|
||||
# --- server -----------------------------------------------------------------
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
"$python" "$server" --root "$(nativepath "$doc")" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- the rewriter in isolation, over a hand-built mirror --------------------
|
||||
printf '[engine self-test] ..\t'
|
||||
st=$(httrack -O /dev/null "-#test=singlefile" "${tmpdir}/st/" 2>/dev/null)
|
||||
case "$st" in
|
||||
*": OK") echo OK ;;
|
||||
*)
|
||||
echo "FAIL: $st"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- crawl ------------------------------------------------------------------
|
||||
# The cap sits between pixel.svg and big.svg, so one inlines and one must not.
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=0)
|
||||
httrack "${common[@]}" --single-file --single-file-max-size 4096 \
|
||||
"${base}/index.html" >"${tmpdir}/log1" 2>&1
|
||||
|
||||
page="${out}/127.0.0.1_${port}/index.html"
|
||||
test -f "$page" || {
|
||||
echo "FAIL: no mirrored page at $page"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- what must be inlined, byte for byte ------------------------------------
|
||||
# The decoder is python, not httrack, so a broken encoder cannot agree with
|
||||
# itself here.
|
||||
extract() {
|
||||
"$python" - "$1" "$2" "$3" <<'PY'
|
||||
import base64, re, sys
|
||||
html = open(sys.argv[1], "rb").read().decode("utf-8", "replace")
|
||||
m = re.search(r'data:%s;base64,([A-Za-z0-9+/=]*)' % re.escape(sys.argv[2]), html)
|
||||
if m is None:
|
||||
sys.exit(1)
|
||||
open(sys.argv[3], "wb").write(base64.b64decode(m.group(1)))
|
||||
PY
|
||||
}
|
||||
|
||||
printf '[image inlined and identical] ..\t'
|
||||
extract "$page" "image/svg+xml" "${tmpdir}/got.svg" || {
|
||||
echo "FAIL: no inlined image"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/pixel.svg" "${tmpdir}/got.svg" || {
|
||||
echo "FAIL: inlined image differs from the served file"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[script inlined and identical] ..\t'
|
||||
extract "$page" "application/x-javascript" "${tmpdir}/got.js" || {
|
||||
echo "FAIL: no inlined script"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/app.js" "${tmpdir}/got.js" || {
|
||||
echo "FAIL: inlined script differs from the served file"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[stylesheet inlined, recursively] ..\t'
|
||||
extract "$page" "text/css" "${tmpdir}/got.css" || {
|
||||
echo "FAIL: no inlined stylesheet"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'url(data:image/svg+xml;base64,' "${tmpdir}/got.css" || {
|
||||
echo "FAIL: url() inside the inlined stylesheet was not inlined"
|
||||
exit 1
|
||||
}
|
||||
extract "${tmpdir}/got.css" "text/css" "${tmpdir}/got2.css" || {
|
||||
echo "FAIL: @import was not inlined"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'url(data:image/svg+xml;base64,' "${tmpdir}/got2.css" || {
|
||||
echo "FAIL: url() inside the @import'ed stylesheet was not inlined"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- what must keep its link ------------------------------------------------
|
||||
printf '[page links stay relative] ..\t'
|
||||
grep -q 'href="other.html"' "$page" || {
|
||||
echo "FAIL: the link to another page was not left relative"
|
||||
exit 1
|
||||
}
|
||||
test -f "${out}/127.0.0.1_${port}/other.html" || {
|
||||
echo "FAIL: the linked page is missing from the mirror"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[over-cap asset stays a link] ..\t'
|
||||
grep -q 'src="big.svg"' "$page" || {
|
||||
echo "FAIL: the over-cap asset did not keep its link"
|
||||
exit 1
|
||||
}
|
||||
test -f "${out}/127.0.0.1_${port}/big.svg" || {
|
||||
echo "FAIL: the over-cap asset is missing from the mirror"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'over the 4096-byte cap' "${out}/hts-log.txt" || {
|
||||
echo "FAIL: the over-cap asset was not reported in the log"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
printf '[the mirror still works] ..\t'
|
||||
if grep -q 'href="style.css"' "$page"; then
|
||||
echo "FAIL: the stylesheet link survived"
|
||||
exit 1
|
||||
fi
|
||||
test -f "${out}/127.0.0.1_${port}/style.css" || {
|
||||
echo "FAIL: the mirror lost the standalone stylesheet"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- a second run must not re-encode ----------------------------------------
|
||||
printf '[idempotent under --update] ..\t'
|
||||
# HTTrack stamps the time into its own footer comment, so compare without it.
|
||||
grep -v 'Mirrored from' "$page" >"${tmpdir}/page1.html"
|
||||
httrack "${common[@]}" --single-file --single-file-max-size 4096 --update \
|
||||
"${base}/index.html" >"${tmpdir}/log2" 2>&1
|
||||
grep -v 'Mirrored from' "$page" >"${tmpdir}/page2.html"
|
||||
cmp -s "${tmpdir}/page1.html" "${tmpdir}/page2.html" || {
|
||||
echo "FAIL: the second run changed the page"
|
||||
exit 1
|
||||
}
|
||||
# A double-encoded payload would decode to another data: URI, not to the SVG.
|
||||
extract "$page" "image/svg+xml" "${tmpdir}/got2.svg" || {
|
||||
echo "FAIL: no inlined image after the second run"
|
||||
exit 1
|
||||
}
|
||||
cmp -s "${doc}/pixel.svg" "${tmpdir}/got2.svg" || {
|
||||
echo "FAIL: the second run re-encoded the inlined image"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
|
||||
# --- the web GUI must emit the same options ---------------------------------
|
||||
printf '[webhttrack renders the options] ..\t'
|
||||
command -v htsserver >/dev/null || {
|
||||
echo "no htsserver in PATH"
|
||||
exit 1
|
||||
}
|
||||
distdir=$(cd "${top_srcdir}" && pwd)
|
||||
sport=$("$python" -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()')
|
||||
srvlog="${tmpdir}/htsserver.log"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
export HOME="${tmpdir}/home"
|
||||
mkdir -p "$HOME"
|
||||
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
|
||||
) &
|
||||
htssrv=$!
|
||||
url=
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "$srvlog") && test -n "$url" && break
|
||||
kill -0 "$htssrv" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || {
|
||||
echo "FAIL: htsserver did not start: $(cat "$srvlog")"
|
||||
exit 1
|
||||
}
|
||||
"$python" - "$url" <<'PY' || exit 1
|
||||
import re, sys, urllib.parse, urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
form = urllib.request.urlopen(url + "server/index.html", timeout=20).read()
|
||||
m = re.search(rb'name="sid" value="([0-9a-f]+)"', form)
|
||||
if not m:
|
||||
sys.exit("FAIL: no session id in server/index.html")
|
||||
fields = [("sid", m.group(1).decode()), ("singlefile", "on"),
|
||||
("singlefilemax", "5000")]
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v)) for k, v in fields)
|
||||
req = urllib.request.Request(url + "server/step4.html",
|
||||
data=body.encode("latin-1"), method="POST")
|
||||
page = urllib.request.urlopen(req, timeout=20).read().decode("latin-1")
|
||||
|
||||
|
||||
def textarea(name):
|
||||
m = re.search(r'<textarea name="%s".*?>(.*?)</textarea>' % name, page, re.S)
|
||||
if m is None:
|
||||
sys.exit("FAIL: no %s textarea in the rendered step4.html" % name)
|
||||
return m.group(1)
|
||||
|
||||
|
||||
cmd = textarea("command")
|
||||
for want in ("--single-file", "--single-file-max-size=5000"):
|
||||
if want not in cmd:
|
||||
sys.exit("FAIL: %s missing from the command line\n%s" % (want, cmd))
|
||||
ini = textarea("winprofile").replace("\r\n", "\n")
|
||||
for want in ("\nSingleFile=1\n", "\nSingleFileMaxSize=5000\n"):
|
||||
if want not in ini:
|
||||
sys.exit("FAIL: %r missing from winprofile.ini\n%s" % (want, ini))
|
||||
PY
|
||||
echo OK
|
||||
|
||||
printf '[project reload maps the keys back] ..\t'
|
||||
grep -q 'do:copy:SingleFile:singlefile' "${distdir}/html/server/step2.html" || {
|
||||
echo "FAIL: step2.html does not restore SingleFile"
|
||||
exit 1
|
||||
}
|
||||
grep -q 'do:copy:SingleFileMaxSize:singlefilemax' "${distdir}/html/server/step2.html" || {
|
||||
echo "FAIL: step2.html does not restore SingleFileMaxSize"
|
||||
exit 1
|
||||
}
|
||||
echo OK
|
||||
@@ -1,148 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The wizard renders its httrack command line as one string, which the engine
|
||||
# splits back into argv: a double quote in a field must reach the split escaped,
|
||||
# or the rest of the value is parsed as fresh options (-V runs a shell command).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
command -v python3 >/dev/null || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
srv=
|
||||
log=$(mktemp)
|
||||
cleanup() {
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
rm -f "${log}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()'
|
||||
}
|
||||
|
||||
# Echo the announced URL. Runs in a command substitution, so it is a
|
||||
# subshell and cannot export the pid: the caller reads it back with srvpid.
|
||||
start() {
|
||||
local port url
|
||||
port=$(freeport)
|
||||
: >"${log}"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
|
||||
) &
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
|
||||
echo "${url}"
|
||||
}
|
||||
|
||||
srvpid() { sed -n 's/^PID=//p' "${log}" | head -1; }
|
||||
|
||||
portof() { echo "${1##*:}" | tr -d /; }
|
||||
|
||||
# Raw request to 127.0.0.1:$1; $2 is the body ("" for a GET of the $3 page,
|
||||
# default index). Prints the reply.
|
||||
request() {
|
||||
python3 -c 'import socket, sys
|
||||
port, body = int(sys.argv[1]), sys.argv[2]
|
||||
page = sys.argv[3] if len(sys.argv) > 3 else "index"
|
||||
if body:
|
||||
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
|
||||
"Content-type: application/x-www-form-urlencoded\r\n"
|
||||
"Content-length: %d\r\n\r\n%s" % (len(body), body))
|
||||
else:
|
||||
req = ("GET /server/%s.html HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % page)
|
||||
s = socket.create_connection(("127.0.0.1", port), 10)
|
||||
s.settimeout(30)
|
||||
s.sendall(req.encode())
|
||||
out = b""
|
||||
while True:
|
||||
b = s.recv(65536)
|
||||
if not b:
|
||||
break
|
||||
out += b
|
||||
s.close()
|
||||
sys.stdout.write(out.decode("latin-1"))' "$1" "$2" ${3+"$3"}
|
||||
}
|
||||
|
||||
scrape_sid() {
|
||||
request "$1" "" | sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1
|
||||
}
|
||||
|
||||
# urlencode the key=value pairs given as arguments
|
||||
formencode() {
|
||||
python3 -c 'import sys, urllib.parse
|
||||
print(urllib.parse.urlencode([tuple(a.split("=", 1)) for a in sys.argv[1:]]))' "$@"
|
||||
}
|
||||
|
||||
url=$(start)
|
||||
port=$(portof "${url}")
|
||||
srv=$(srvpid)
|
||||
test -n "${srv}" || fail "htsserver did not report its pid"
|
||||
|
||||
sid=$(scrape_sid "${port}")
|
||||
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid from the page (got '${sid}')"
|
||||
|
||||
# Fill the wizard fields the command line quotes, then read back the generated
|
||||
# command line: the user-agent carries a break-out attempt, the footer a
|
||||
# backslash (which must survive the escape round trip), the project name a plain
|
||||
# value.
|
||||
body=$(formencode "sid=${sid}" 'user=Moz" -V "touch /tmp/pwn' 'footer=a\b"c' \
|
||||
"path=/tmp/p" "projname=plain proj" 'urls=http://x/a"b' 'url2=+*.png"')
|
||||
request "${port}" "${body}" >/dev/null
|
||||
cmdline=$(request "${port}" "" step4 |
|
||||
sed -n '/<textarea name="command"/,/<\/textarea>/p')
|
||||
|
||||
# Control: without the fields in the page every assertion below is vacuous.
|
||||
grep -q -- '--user-agent' <<<"${cmdline}" ||
|
||||
fail "no --user-agent in the generated command line (probe blind)"
|
||||
|
||||
# A plain value is passed through untouched: the escaping must not mangle the
|
||||
# ordinary case.
|
||||
grep -qF -- '--path "/tmp/p/plain proj"' <<<"${cmdline}" ||
|
||||
fail "a plain quoted value was not passed through: ${cmdline}"
|
||||
|
||||
# The quote is escaped, so the split keeps it inside the value...
|
||||
grep -qF -- '--user-agent "Moz\" -V \"touch /tmp/pwn"' <<<"${cmdline}" ||
|
||||
fail "the quote in the user-agent was not escaped: ${cmdline}"
|
||||
|
||||
# ...and the raw form, which would end the argument and hand -V to the option
|
||||
# parser, is gone.
|
||||
grep -qF -- '--user-agent "Moz" -V "touch' <<<"${cmdline}" &&
|
||||
fail "the user-agent still closes its argument early: ${cmdline}"
|
||||
|
||||
# A backslash is escaped too, or the split would eat it along with the quote
|
||||
# that follows.
|
||||
grep -qF -- '--footer "a\\b\"c"' <<<"${cmdline}" ||
|
||||
fail "the backslash in the footer was not escaped: ${cmdline}"
|
||||
|
||||
# The url and filter fields sit outside quotes, where a backslash cannot escape
|
||||
# anything: one quote there flips the parity of every quote after it, so the
|
||||
# escaping above would protect nothing. They must not emit a raw quote at all.
|
||||
grep -qF -- 'http://x/a%22b' <<<"${cmdline}" ||
|
||||
fail "the quote in the url field was not neutralised: ${cmdline}"
|
||||
grep -qF -- '+*.png%22' <<<"${cmdline}" ||
|
||||
fail "the quote in the filter field was not neutralised: ${cmdline}"
|
||||
grep -qF -- 'http://x/a"b' <<<"${cmdline}" &&
|
||||
fail "the url field still emits a raw quote: ${cmdline}"
|
||||
|
||||
echo "PASS"
|
||||
@@ -1,161 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# htsserver serves the crawled mirror under /website/ alongside its own GUI:
|
||||
# mirrored pages must skip the ${...} expander, or ${_sid} leaks the session id.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
command -v python3 >/dev/null || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
log=$(mktemp)
|
||||
work=$(mktemp -d)
|
||||
csrv=
|
||||
# start() runs in a command substitution, so its $! never reaches this shell. A
|
||||
# missed kill leaves an orphan holding the CI job open long after a green suite.
|
||||
srvpid() { sed -n 's/^PID=//p' "${log}" 2>/dev/null | head -1; }
|
||||
cleanup() {
|
||||
local pid
|
||||
pid=$(srvpid)
|
||||
test -z "${pid}" || kill -9 "${pid}" 2>/dev/null || true
|
||||
test -z "${csrv}" || kill -9 "${csrv}" 2>/dev/null || true
|
||||
wait "${csrv}" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
rm -rf "${log}" "${work}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()'
|
||||
}
|
||||
|
||||
# Echo the announced URL. Extra args become key/value pairs in the server store,
|
||||
# which is how the project path gets set here; the GUI posts it from step4.
|
||||
start() {
|
||||
local port url
|
||||
port=$(freeport)
|
||||
: >"${log}"
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
exec htsserver "${distdir}/" --port "${port}" "$@" >"${log}" 2>&1
|
||||
) &
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
|
||||
echo "${url}"
|
||||
}
|
||||
|
||||
portof() { echo "${1##*:}" | tr -d /; }
|
||||
|
||||
# GET $2 from 127.0.0.1:$1, headers into $3 and the body, byte for byte, into $4.
|
||||
fetch() {
|
||||
python3 -c 'import socket, sys
|
||||
s = socket.create_connection(("127.0.0.1", int(sys.argv[1])), 10)
|
||||
s.settimeout(20)
|
||||
s.sendall(("GET %s HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % sys.argv[2]).encode())
|
||||
out = b""
|
||||
while True:
|
||||
b = s.recv(65536)
|
||||
if not b:
|
||||
break
|
||||
out += b
|
||||
s.close()
|
||||
head, _, body = out.partition(b"\r\n\r\n")
|
||||
open(sys.argv[3], "wb").write(head)
|
||||
open(sys.argv[4], "wb").write(body)' "$1" "$2" "$3" "$4"
|
||||
}
|
||||
|
||||
# POST the remaining args as urlencoded key=value fields to 127.0.0.1:$1.
|
||||
post() {
|
||||
local port=$1
|
||||
shift
|
||||
python3 -c 'import socket, sys, urllib.parse
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe=""))
|
||||
for k, v in (a.split("=", 1) for a in sys.argv[2:])).encode()
|
||||
s = socket.create_connection(("127.0.0.1", int(sys.argv[1])), 10)
|
||||
s.settimeout(30)
|
||||
s.sendall(b"POST /step4.html HTTP/1.0\r\nHost: 127.0.0.1\r\n"
|
||||
b"Content-type: application/x-www-form-urlencoded\r\n"
|
||||
b"Content-length: %d\r\n\r\n" % len(body) + body)
|
||||
while s.recv(65536):
|
||||
pass
|
||||
s.close()' "${port}" "$@" >/dev/null
|
||||
}
|
||||
|
||||
# LF-only, and a line ending in a backslash: the expander rewrites both, so a
|
||||
# mangled reply fails the byte comparison even where no directive is present.
|
||||
mirror="${work}/hostile.html"
|
||||
# shellcheck disable=SC2016 # the directives are the payload, not shell expansions
|
||||
printf 'Hostile mirrored page.\nsid=${_sid} copy=${sid}\ntrailing backslash: \\\n' \
|
||||
>"${mirror}"
|
||||
|
||||
hdr="${work}/hdr"
|
||||
body="${work}/body"
|
||||
|
||||
# The server merges $HOME/.httrack.ini into the same store on the first request;
|
||||
# point it somewhere empty so a developer's own file cannot shadow projpath.
|
||||
export HOME="${work}"
|
||||
|
||||
url=$(start projpath "${work}/")
|
||||
port=$(portof "${url}")
|
||||
|
||||
# Positive control: the GUI's own templates must still expand. This is also
|
||||
# where the real token comes from, so its absence below can be asserted.
|
||||
fetch "${port}" /server/index.html "${hdr}" "${body}"
|
||||
sid=$(sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' "${body}" | head -1)
|
||||
test "${#sid}" -eq 32 || fail "GUI page did not expand \${sid} (got '${sid}')"
|
||||
|
||||
fetch "${port}" /website/hostile.html "${hdr}" "${body}"
|
||||
grep -q '^HTTP/1\.0 200 ' "${hdr}" || fail "mirrored page not served: $(head -1 "${hdr}")"
|
||||
grep -qF "${sid}" "${body}" &&
|
||||
fail "the session id was expanded into mirrored content"
|
||||
# shellcheck disable=SC2016 # the directives are the payload, not shell expansions
|
||||
grep -qF '${_sid}' "${body}" || fail "\${_sid} did not survive verbatim"
|
||||
# shellcheck disable=SC2016
|
||||
grep -qF '${sid}' "${body}" || fail "\${sid} did not survive verbatim"
|
||||
cmp -s "${mirror}" "${body}" || fail "mirrored file not served byte for byte"
|
||||
# The mirror stays browsable: verbatim must not mean served as a download.
|
||||
grep -qi '^Content-type: text/html' "${hdr}" ||
|
||||
fail "mirrored page lost its text/html type: $(cat "${hdr}")"
|
||||
|
||||
# The other direction: while a crawl runs, every .html request is overridden to
|
||||
# the GUI's own refresh page, so a /website/ URL stops naming mirrored content.
|
||||
# /trickle/ dribbles for a minute, which holds the crawl open for the probe.
|
||||
clog="${work}/content.log"
|
||||
python3 "${testdir}/local-server.py" --root "${work}" >"${clog}" 2>&1 &
|
||||
csrv=$!
|
||||
for _ in $(seq 1 40); do
|
||||
cport=$(sed -n 's/^PORT //p' "${clog}") && test -n "${cport}" && break
|
||||
kill -0 "${csrv}" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${cport:-}" || fail "content server did not come up: $(cat "${clog}")"
|
||||
|
||||
post "${port}" "sid=${sid}" "path=${work}" projname=crawl winprofile=x \
|
||||
command_do=start \
|
||||
"command=httrack --quiet --robots=0 http://127.0.0.1:${cport}/trickle/ -O ${work}/crawl"
|
||||
|
||||
fetch "${port}" /website/hostile.html "${hdr}" "${body}"
|
||||
grep -q '^HTTP/1\.0 200 ' "${hdr}" ||
|
||||
fail "running crawl: /website/ was not overridden to the GUI page: $(head -1 "${hdr}")"
|
||||
grep -qF "'crawl' - HTTrack Website Copier" "${body}" ||
|
||||
fail "the overridden GUI page was not the expanded refresh page"
|
||||
|
||||
echo "PASS"
|
||||
@@ -32,7 +32,6 @@ TESTS = \
|
||||
00_runnable.test \
|
||||
01_engine-charset.test \
|
||||
01_engine-cmdline.test \
|
||||
01_engine-cmdline-split.test \
|
||||
01_engine-cookies.test \
|
||||
01_engine-copyopt.test \
|
||||
01_engine-crange.test \
|
||||
@@ -65,7 +64,6 @@ TESTS = \
|
||||
01_engine-reconcile.test \
|
||||
01_engine-expandhome.test \
|
||||
01_engine-fsize.test \
|
||||
01_engine-growsize.test \
|
||||
01_engine-redirect.test \
|
||||
01_engine-longpath-io.test \
|
||||
01_engine-mirror-io.test \
|
||||
@@ -149,7 +147,6 @@ TESTS = \
|
||||
50_local-contentcodings.test \
|
||||
51_local-update-codec.test \
|
||||
52_local-socks5.test \
|
||||
53_local-proxytrack-arc-reason.test \
|
||||
53_local-proxytrack-cache-corrupt.test \
|
||||
54_local-update-truncate-purge.test \
|
||||
55_local-chunked.test \
|
||||
@@ -173,14 +170,6 @@ TESTS = \
|
||||
74_local-warc-wacz.test \
|
||||
74_local-warc-verbatim.test \
|
||||
75_engine-longpath-posix.test \
|
||||
76_cli-resize.test \
|
||||
77_webhttrack-redirect.test \
|
||||
78_webhttrack-sid.test \
|
||||
79_local-proxytrack-webdav-mime.test \
|
||||
80_engine-crash-symbolize.test \
|
||||
81_webhttrack-maxsize.test \
|
||||
82_local-single-file.test \
|
||||
83_webhttrack-argescape.test \
|
||||
84_webhttrack-mirror-verbatim.test
|
||||
76_cli-resize.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -46,15 +46,11 @@ cat >"$stubdir/x-www-browser" <<EOF
|
||||
echo "stub browser invoked with: \$1" >&2
|
||||
# Also fetch an option page and require a rendered title='' tooltip: proves the
|
||||
# option template expands and the \${html:} filter escapes into the attribute.
|
||||
# option9 additionally proves the WARC control renders with its expanded label,
|
||||
# and option2 the --single-file pair: on field names plus the absence of an
|
||||
# unexpanded key, since the default locale here is French.
|
||||
# option9 additionally proves the WARC control renders with its expanded label.
|
||||
opturl="\${1%/}/server/option2.html"
|
||||
warcurl="\${1%/}/server/option9.html"
|
||||
if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html &&
|
||||
opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='" &&
|
||||
printf '%s' "\$opt" | grep -qaF 'name="singlefile"' && printf '%s' "\$opt" | grep -qaF 'name="singlefilemax"' &&
|
||||
! printf '%s' "\$opt" | grep -qaF '\${LANG_SINGLEFILE}' &&
|
||||
warc="\$(curl -fsSL --max-time 20 "\$warcurl")" && printf '%s' "\$warc" | grep -qaF 'name="warcfile"' && printf '%s' "\$warc" | grep -qaF WARC; then
|
||||
echo PASS >"$marker"
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user