Compare commits

..

2 Commits

Author SHA1 Message Date
Xavier Roche
5a043ccf60 Size the new strsafe buffers away from sizeof(char*)
char[8] equals a pointer on LP64, so MSVC's array-vs-pointer heuristic
read the unterminated source as a pointer, skipped the bound and never
aborted; the x64 build failed while Win32 passed. Same trap the existing
comment in that function already warns about.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-26 10:24:55 +02:00
Xavier Roche
2193754d9e ARC cache replays a truncated HTTP reason phrase
proxytrack bounded the reason-phrase copy out of an ARC index by
sizeof(pos) - 1 where pos is a const char *, so a stored "404 Not Found"
replays as "404 Not Fou" (and "404 Not" on 32-bit). Use strncatbuff with
the destination's own size.

Fold the nine copies of the buff() family's source-capacity expression
into HTS_SIZEOF_SRC_, applying sizeof to the type so a decayed operand no
longer trips -Wsizeof-array-decay at five call sites; MSVC keeps the old
expression behind the guard HTS_IS_CHAR_BUFFER already uses. The two
other raw strncat calls become strncatbuff, htsbuff_catn stops handing
strnlen the (size_t)-1 sentinel, and htsweb.c no longer compares ep
against a NULL eps.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-26 10:13:09 +02:00
45 changed files with 322 additions and 2650 deletions

View File

@@ -225,9 +225,8 @@ jobs:
# Every gate here exits 77, so an all-skipped suite would report green having
# tested nothing: pin the skips, and floor the passes in case the glob empties.
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
# webdav-mime needs a reapable background listener, which MSYS cannot give it.
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test"
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581.
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test"
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }

View File

@@ -66,11 +66,10 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
AC_SUBST(VERSION_INFO)
### Default CFLAGS
# No -Wdeclaration-after-statement: nothing sets -std=, so this builds as gnu17.
DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
-Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
-Wstrict-prototypes -Wmissing-prototypes \
-Wmissing-declarations \
-Wmissing-declarations -Wdeclaration-after-statement \
-Wpointer-arith -Wsequence-point -Wnested-externs \
-D_REENTRANT"
AC_SUBST(DEFAULT_CFLAGS)
@@ -78,30 +77,27 @@ DEFAULT_LDFLAGS=""
AC_SUBST(DEFAULT_LDFLAGS)
### Additional flags (if supported)
# -Werror on probes: exit-status-only checks let clang's warn-on-unknown-flag through.
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"], [], [-Werror])
# -Wformat-nonliteral needs -Wformat in the probe or gcc rejects it as ignored.
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"], [], [-Werror -Wformat])
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"])
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"])
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"])
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"])
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"])
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"])
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"])
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"])
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"])
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"])
# Make htssafe.h's pointer-dest 'warning' attribute a hard error in our build
# (migration is at zero; a new char* dest is a regression). gcc/clang each take
# only their own spelling; downstream keeps the plain warning, not a build break.
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"], [], [-Werror])], [-Werror])
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"], [], [-Werror])
# No --discard-all: it drops the local symbols naming every static function, so
# a trace misattributes them to the nearest surviving global. Costs 0.6% size.
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--no-undefined"])
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
@@ -123,13 +119,13 @@ AC_SUBST([LIBC_FORCE_LINK])
### PIE
CFLAGS_PIE=""
LDFLAGS_PIE=""
AX_CHECK_COMPILE_FLAG([-fpie], [CFLAGS_PIE="-fpie"], [], [-Werror])
AX_CHECK_COMPILE_FLAG([-fpie -pie], [CFLAGS_PIE="-fpie -pie"])
AX_CHECK_LINK_FLAG([-pie], [LDFLAGS_PIE="-pie"])
AC_SUBST([CFLAGS_PIE])
AC_SUBST([LDFLAGS_PIE])
# Ties a crash trace from a stripped build back to its separate debug symbols.
AX_CHECK_LINK_FLAG([-Wl,--build-id], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--build-id"])
## Export all symbols for backtraces
AX_CHECK_COMPILE_FLAG([-rdynamic], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -rdynamic"])
### Check for -fvisibility=hidden support
gl_VISIBILITY

View File

@@ -97,8 +97,9 @@ ${do:end-if}
</pre>
${LANG_G8} :
${/* an http: page cannot navigate to file:, so the mirror is reached through the server */}
<a href="/website/index.html" target="_new">
${do:output-mode:html-urlescaped}
<a href="file://${path}/${projname}/" target="_new">
${do:output-mode:}
${path}/${projname}
</a></li>
<ul>

View File

@@ -103,7 +103,7 @@ ${LANG_Q3}
<tr><td>
<table width="100%">
<tr><td align="left">
<input type="submit" value="${LANG_OK}"
<input type="submit" value="${LANG_OK]"
${do:output-mode:html-urlescaped}
onClick="if (confirm(str_replace(str_replace('${LANG_DIAL7}', '%20', ' '), '%0a', ' '))) { form.closeme.value=1; form.submit(); } return false;"
${do:output-mode:}

View File

@@ -121,9 +121,9 @@ httrack \
--quiet \
--build-top-index \
${test:todo:--mirror:--mirror:--mirror-wizard:--get:--mirrorlinks:--testlinks:--continue:--update}
${unquoted:urls}
${test:filelist:-%L "}${arg:filelist}${test:filelist:"}
--path "${arg:path}/${arg:projname}"
${urls}
${test:filelist:-%L "}${filelist}${test:filelist:"}
--path "${html:path}/${html:projname}"
\
${test:parseall:--near}
${test:link:--test}
@@ -131,7 +131,7 @@ httrack \
${test:htmlfirst::--priority=7}
\
${do:if-not-empty:BuildString}
--structure "${arg:BuildString}"
--structure "${BuildString}"
${do:end-if}
${test:build:-N0:-N0:-N1:-N2:-N3:-N4:-N5:-N100:-N101:-N102:-N103:-N104:-N105:-N99:-N199:}
\
@@ -150,30 +150,29 @@ ${do:end-if}
${test:travel3::--keep-links=0:--keep-links:--keep-links=3:--keep-links=4}
${test:windebug:--debug-headers}
\
${test:connexion:--sockets=}${unquoted:connexion}
${test:connexion:--sockets=}${connexion}
${test:ka:--keep-alive}
${test:timeout:--timeout=}${unquoted:timeout}
${test:timeout:--timeout=}${timeout}
${test:remt:--host-control=1}
${test:retry:--retries=}${unquoted:retry}
${test:rate:--min-rate=}${unquoted:rate}
${test:retry:--retries=}${retry}
${test:rate:--min-rate=}${rate}
${test:rems:--host-control=2}
\
${test:depth:--depth=}${unquoted:depth}
${test:depth2:--ext-depth=}${unquoted:depth2}
${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one */}
${test:othermax:--max-files=}${unquoted:othermax}
${test:maxhtml:--max-files=,}${unquoted:maxhtml}
${test:sizemax:--max-size=}${unquoted:sizemax}
${test:pausebytes:--max-pause=}${unquoted:pausebytes}
${test:maxtime:--max-time=}${unquoted:maxtime}
${test:maxrate:--max-rate=}${unquoted:maxrate}
${test:maxconn:--connection-per-second=}${unquoted:maxconn}
${test:maxlinks:--advanced-maxlinks=}${unquoted:maxlinks}
${test:depth:--depth=}${depth}
${test:depth2:--ext-depth=}${depth2}
${test:maxhtml:--max-files=,}${maxhtml}
${test:othermax:--max-files=}${othermax}
${test:sizemax:--max-files=}${sizemax}
${test:pausebytes:--max-pause=}${pausebytes}
${test:maxtime:--max-time=}${maxtime}
${test:maxrate:--max-rate=}${maxrate}
${test:maxconn:--connection-per-second=}${maxconn}
${test:maxlinks:--advanced-maxlinks=}${maxlinks}
\
--user-agent "${arg:user}"
--footer "${arg:footer}"
--user-agent "${html:user}"
--footer "${html:footer}"
\
${unquoted:url2}
${url2}
\
${test:cookies:--cookies=0:}
${test:parsejava:--parse-java=0:}
@@ -182,20 +181,20 @@ ${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one *
${test:keepwww:--keep-www-prefix}
${test:keepslashes:--keep-double-slashes}
${test:keepqueryorder:--keep-query-order}
${test:cookiesfile:--cookies-file "}${arg:cookiesfile}${test:cookiesfile:"}
${test:pausefiles:--pause "}${arg:pausefiles}${test:pausefiles:"}
${test:stripquery:--strip-query "}${arg:stripquery}${test:stripquery:"}
${test:cookiesfile:--cookies-file "}${html:cookiesfile}${test:cookiesfile:"}
${test:pausefiles:--pause "}${pausefiles}${test:pausefiles:"}
${test:stripquery:--strip-query "}${html:stripquery}${test:stripquery:"}
${test:toler:--tolerant}
${test:http10:--http-10}
${test:cache2:--store-all-in-cache}
${test:warc:--warc}
${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
${test:warcfile:--warc-file "}${html:warcfile}${test:warcfile:"}
${test:norecatch:--do-not-recatch}
${test:logf:--single-log}
${test:logtype:::--extra-log:--debug-log}
${test:index:--index=0:}
${test:index2:--search-index=0:--search-index}
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${arg:prox}${test:prox:\3A}${arg:portprox}${test:prox:"}
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
</textarea>
@@ -214,7 +213,7 @@ ParseAll=${ztest:parseall:0:1}
HTMLFirst=${ztest:htmlfirst:0:1}
Cache=${ztest:cache:0:1}
NoRecatch=${ztest:norecatch:0:1}
Dos=${dos}
Dos=${dos
Index=${ztest:index:0:1}
WordIndex=${ztest:index2:0:1}
Log=${ztest:logf:0:1:2}

View File

@@ -32,9 +32,7 @@ AM_LDFLAGS = \
bin_PROGRAMS = proxytrack httrack htsserver
httrack_SOURCES = httrack.c htsbacktrace.c htsbacktrace.h
# $(DL_LIBS): dladdr() in the crash handler, still in libdl on pre-2.34 glibc.
httrack_LDADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
httrack_LDADD = $(THREADS_LIBS) libhttrack.la
htsserver_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) libhttrack.la
proxytrack_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS)
@@ -49,7 +47,6 @@ htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
lib_LTLIBRARIES = libhttrack.la
htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h \
htscmdline.c htscmdline.h \
htsurlport.c htsurlport.h
proxytrack_SOURCES = proxy/main.c \
proxy/proxytrack.c proxy/store.c \
@@ -63,7 +60,7 @@ whttrackrun_SCRIPTS = webhttrack
libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
htscache_selftest.c htsdns_selftest.c htsselftest.c \
htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
htshelp.c htslib.c htsurlport.c htscoremain.c \
htsname.c htsrobots.c htstools.c htswizard.c \
htsalias.c htsthread.c htsindex.c htsbauth.c \
htsmd5.c htscodec.c htswarc.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
@@ -72,7 +69,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h \
htscmdline.h htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
htsmodules.h htsname.h htsnet.h htssniff.h \

View File

@@ -541,15 +541,9 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
const char *ext) {
// do not use tempnam() but a regular filename
back->tmpfile_buffer[0] = '\0';
if (back->url_sav[0] != '\0') {
/* same capacity as url_sav, so truncation drops the extension and aliases
the temp name onto the live file that back_finalize_backup() UNLINKs */
if (!sprintfbuff(back->tmpfile_buffer, "%s.%s", back->url_sav, ext)) {
hts_log_print(opt, LOG_WARNING, "temporary filename too long for %s",
back->url_sav);
back->tmpfile_buffer[0] = '\0';
return -1;
}
if (back->url_sav != NULL && back->url_sav[0] != '\0') {
snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
back->url_sav, ext);
back->tmpfile = back->tmpfile_buffer;
if (structcheck(back->tmpfile) != 0) {
hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
@@ -557,15 +551,8 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
return -1;
}
} else {
/* truncation here would collide distinct tmpnameid's onto one name */
if (!sprintfbuff(back->tmpfile_buffer, "%s/tmp%d.%s",
StringBuff(opt->path_html_utf8), opt->state.tmpnameid++,
ext)) {
hts_log_print(opt, LOG_WARNING, "temporary filename too long in %s",
StringBuff(opt->path_html_utf8));
back->tmpfile_buffer[0] = '\0';
return -1;
}
snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
back->tmpfile = back->tmpfile_buffer;
}
/* OK */
@@ -900,7 +887,8 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
HTS_STAT.stat_bytes += back[p].r.size;
HTS_STAT.stat_files++;
hts_log_print(opt, LOG_TRACE, "added file %s%s => %s",
back[p].url_adr, back[p].url_fil, back[p].url_sav);
back[p].url_adr, back[p].url_fil,
back[p].url_sav != NULL ? back[p].url_sav : "");
}
if ((!back[p].r.notmodified) && (opt->is_update)) {
HTS_STAT.stat_updated_files++; // page modifiée
@@ -2314,24 +2302,26 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
&& slot_can_be_finalized(opt, &back[i]);
int may_serialize = slot_can_be_cached_on_disk(&back[i]);
hts_log_print(
opt, LOG_DEBUG,
"back[%03d]: may_clean=%d, may_finalize_disk=%d, "
"may_serialize=%d:" LF "\t"
"finalized(%d), status(%d), locked(%d), delayed(%d), "
"test(%d), " LF "\t"
"statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), " LF
"\t"
"contenttype(%s), url(%s%s), save(%s)",
i, may_clean, may_finalize, may_serialize, back[i].finalized,
back[i].status, back[i].locked, IS_DELAYED_EXT(back[i].url_sav),
back[i].testmode, back[i].r.statuscode, (int) back[i].r.size,
back[i].r.is_write,
may_be_hypertext_mime(opt, back[i].r.contenttype,
back[i].url_fil),
/* */
back[i].r.contenttype, back[i].url_adr, back[i].url_fil,
back[i].url_sav);
hts_log_print(opt, LOG_DEBUG,
"back[%03d]: may_clean=%d, may_finalize_disk=%d, may_serialize=%d:"
LF "\t"
"finalized(%d), status(%d), locked(%d), delayed(%d), test(%d), "
LF "\t"
"statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), "
LF "\t" "contenttype(%s), url(%s%s), save(%s)", i,
may_clean, may_finalize, may_serialize,
back[i].finalized, back[i].status, back[i].locked,
IS_DELAYED_EXT(back[i].url_sav), back[i].testmode,
back[i].r.statuscode, (int) back[i].r.size,
back[i].r.is_write, may_be_hypertext_mime(opt,
back[i].r.
contenttype,
back[i].
url_fil),
/* */
back[i].r.contenttype, back[i].url_adr,
back[i].url_fil,
back[i].url_sav ? back[i].url_sav : "<null>");
}
}
}
@@ -2867,8 +2857,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
// new session
back[i].r.ssl_con = SSL_new(openssl_ctx);
if (back[i].r.ssl_con) {
/* non-const twin: the OpenSSL macro casts the qualifier away */
char *hostname = jump_protocol(back[i].url_adr);
const char* hostname = jump_protocol_const(back[i].url_adr);
// some servers expect the hostname on the clienthello (SNI TLS extension)
SSL_set_tlsext_host_name(back[i].r.ssl_con, hostname);
SSL_clear(back[i].r.ssl_con);

View File

@@ -1,238 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: crash backtrace printer */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
/* Before every header: glibc gates dladdr() on it. */
#if defined(__linux) && !defined(_GNU_SOURCE)
#define _GNU_SOURCE
#endif
#include "htsbacktrace.h"
#include "htsglobal.h"
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
#include <io.h> /* write */
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#if (defined(__linux) && defined(HAVE_EXECINFO_H))
#include <dlfcn.h>
#include <errno.h>
#include <execinfo.h>
#include <signal.h>
#include <time.h>
#include <sys/wait.h>
#define USES_BACKTRACE
#endif
#ifdef USES_BACKTRACE
#define BT_MAX_FRAMES 64 /* frames we try to name */
#define BT_MAX_MODULES 8 /* distinct modules, one child each */
#define BT_HEX_SIZE 19 /* "0x" + 16 nibbles + NUL */
#define BT_PATH_SIZE 1024 /* module path; longer is skipped */
#define BT_WAIT_TICKS 300 /* 10ms ticks, shared: cap a slow child */
#define BT_NO_SYMBOLIZER 127 /* child exit: execvp() found none */
static hts_boolean symbolize_crash = HTS_TRUE;
/* "0x"-prefixed hex: the handler must stay stdio-free. */
static void print_hex(char *buffer, uintptr_t value) {
static const char digits[] = "0123456789abcdef";
size_t i = 2, a, b;
buffer[0] = '0';
buffer[1] = 'x';
do {
buffer[i++] = digits[value & 0xf];
value >>= 4;
} while (value != 0);
buffer[i] = '\0';
for (a = 2, b = i - 1; a < b; a++, b--) {
const char c = buffer[a];
buffer[a] = buffer[b];
buffer[b] = c;
}
}
/* HTS_FALSE if src does not fit: a truncated module path would point the
symbolizer at the wrong file. */
static hts_boolean copy_bounded(char *dest, size_t size, const char *src) {
size_t i;
for (i = 0; i < size - 1 && src[i] != '\0'; i++) {
dest[i] = src[i];
}
dest[i] = '\0';
return src[i] == '\0' ? HTS_TRUE : HTS_FALSE;
}
/* Run the symbolizer on argv, output on fd, within *budget ticks. HTS_FALSE
only if none could be run at all; otherwise silent, the raw trace stands. */
static hts_boolean spawn_symbolizer(char **argv, int fd, int *budget) {
const pid_t pid = fork();
int status = 0;
if (pid == -1)
return HTS_FALSE;
if (pid == 0) {
static char llvm_prog[] = "llvm-symbolizer";
static char llvm_opts[] = "-p";
dup2(fd, 1); /* both symbolizers write on stdout */
execvp(argv[0], argv);
argv[0] = llvm_prog; /* an LLVM-only install ships no addr2line */
argv[1] = llvm_opts;
execvp(argv[0], argv);
_exit(BT_NO_SYMBOLIZER);
}
for (; *budget > 0; (*budget)--) {
const struct timespec tick = {0, 10 * 1000 * 1000};
const pid_t reaped = waitpid(pid, &status, WNOHANG);
if (reaped == pid)
return WIFEXITED(status) && WEXITSTATUS(status) == BT_NO_SYMBOLIZER
? HTS_FALSE
: HTS_TRUE;
if (reaped == -1 && errno != EINTR)
return HTS_TRUE;
nanosleep(&tick, NULL);
}
kill(pid, SIGKILL);
waitpid(pid, NULL, 0);
return HTS_TRUE;
}
/* Name the frames backtrace_symbols_fd() leaves as module+offset:
-fvisibility=hidden keeps them out of .dynsym, but DWARF has them. dladdr()
is not formally async-signal-safe; accepted, this path is already fatal. */
static void symbolize_backtrace(void *const *stack, int size, int fd) {
static char prog[] = "addr2line";
static char opts[] = "-Cfipa";
static char dashe[] = "-e";
char hex[BT_MAX_FRAMES][BT_HEX_SIZE];
const void *base[BT_MAX_FRAMES];
const char *name[BT_MAX_FRAMES];
hts_boolean grouped[BT_MAX_FRAMES];
char module[BT_PATH_SIZE];
char *argv[4 + BT_MAX_FRAMES + 1];
int budget = BT_WAIT_TICKS;
int i, spawned;
if (size > BT_MAX_FRAMES)
size = BT_MAX_FRAMES;
for (i = 0; i < size; i++) {
Dl_info info;
grouped[i] = HTS_TRUE; /* skipped unless dladdr() places the frame */
if (dladdr(stack[i], &info) == 0 || info.dli_fname == NULL ||
info.dli_fname[0] == '\0')
continue;
base[i] = info.dli_fbase;
name[i] = info.dli_fname;
print_hex(hex[i], (uintptr_t) ((const char *) stack[i] -
(const char *) info.dli_fbase));
grouped[i] = HTS_FALSE;
}
/* One child per module: addr2line takes a single -e. Each frame is claimed
once, so argc cannot exceed argv[]. */
for (spawned = 0; spawned < BT_MAX_MODULES; spawned++) {
int first, j, argc = 0;
for (first = 0; first < size && grouped[first]; first++)
;
if (first >= size)
break;
argv[argc++] = prog;
argv[argc++] = opts;
argv[argc++] = dashe;
argv[argc++] = module;
for (j = first; j < size; j++) {
if (grouped[j] || base[j] != base[first])
continue;
grouped[j] = HTS_TRUE;
argv[argc++] = hex[j];
}
argv[argc] = NULL;
/* access(): skip pseudo-modules like linux-vdso, which have no file and
would draw nothing but an addr2line complaint. */
if (copy_bounded(module, sizeof(module), name[first]) &&
access(module, R_OK) == 0) {
const size_t len = strlen(module);
/* addr2line -a prints offsets only: say which module they are in. */
(void) (write(fd, module, len) == (ssize_t) len);
(void) (write(fd, ":\n", 2) == 2);
if (!spawn_symbolizer(argv, fd, &budget))
break; /* no symbolizer: stop at one header */
}
}
}
#endif
void hts_backtrace_init(void) {
#ifdef USES_BACKTRACE
symbolize_crash =
getenv("HTTRACK_NO_SYMBOLIZE") == NULL ? HTS_TRUE : HTS_FALSE;
#endif
}
void hts_print_backtrace(int fd) {
#ifdef USES_BACKTRACE
void *stack[256];
const int size = backtrace(stack, sizeof(stack) / sizeof(stack[0]));
/* A fault inside the handler lands back here: symbolizing twice interleaves
two traces on fd and spends a second budget. */
static volatile sig_atomic_t entered = 0;
if (size != 0) {
backtrace_symbols_fd(stack, size, fd);
if (symbolize_crash && entered == 0) {
entered = 1;
symbolize_backtrace(stack, size, fd);
entered = 0;
}
}
#else
const char msg[] = "No stack trace available on this OS :(\n";
if (write(fd, msg, sizeof(msg) - 1) != sizeof(msg) - 1) {
/* sorry GCC */
}
#endif
}

View File

@@ -1,45 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: crash backtrace printer */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSBACKTRACE_DEFH
#define HTSBACKTRACE_DEFH
/* Sample HTTRACK_NO_SYMBOLIZE before any crash: getenv() is not signal-safe.
Call once, from the process that installs the fatal-signal handlers. */
void hts_backtrace_init(void);
/* Write the calling thread's stack to fd, callable from a fatal signal handler:
raw frames first, then whatever an external symbolizer can name. Allocates
nothing; prints a one-line notice where the OS has no backtrace(). */
void hts_print_backtrace(int fd);
#endif

View File

@@ -629,8 +629,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
// File exists on disk with declared cache name (this is expected!)
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) { // un fichier existe déja
// Expected size ?
const LLint fsize = fsize_utf8(
fconv(catbuff, sizeof(catbuff), previous_save));
const size_t fsize =
fsize_utf8(fconv(catbuff, sizeof(catbuff), previous_save));
if (fsize == r.size) {
// Target name is the previous name, and the file looks good: nothing to do!
if (strcmp(previous_save, target_save) == 0) {
@@ -666,8 +666,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
// Suppose a broken mirror, with a file being renamed: OK
else if (fexist_utf8(fconv(catbuff, sizeof(catbuff), target_save))) {
// Expected size ?
const LLint fsize =
fsize_utf8(fconv(catbuff, sizeof(catbuff), target_save));
const size_t fsize = fsize_utf8(fconv(catbuff, sizeof(catbuff), target_save));
if (fsize == r.size) {
// So far so good
@@ -1441,7 +1440,7 @@ int cache_brstr(char *adr, char *s, size_t s_size) {
/* binput bounded to a NUL-terminated buffer: refuse to start a read at or
past `end`, so a prior over-advance can't walk a cache-index parse OOB. */
int cache_binput(const char *adr, const char *end, char *s, int max) {
int cache_binput(char *adr, const char *end, char *s, int max) {
if (adr >= end) {
s[0] = '\0';
return 0;

View File

@@ -93,7 +93,7 @@ void cache_rstr(FILE *fp, char *s, size_t s_size);
char *cache_rstr_addr(FILE * fp);
int cache_brstr(char *adr, char *s, size_t s_size);
/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
int cache_binput(const char *adr, const char *end, char *s, int max);
int cache_binput(char *adr, const char *end, char *s, int max);
int cache_brint(char *adr, int *i);
void cache_rint(FILE * fp, int *i);
void cache_rLLint(FILE * fp, LLint * i);

View File

@@ -1,95 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: command line splitter, shared by the engine and */
/* htsserver */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#include "htscmdline.h"
#include "htssafe.h"
#include <limits.h>
#include <stdint.h>
char **hts_split_cmdline(char *cmd, int *nargs) {
size_t nsep = 0;
size_t capacity;
size_t r;
size_t w;
int argc = 0;
hts_boolean quoted = HTS_FALSE;
char **argv;
*nargs = 0;
/* fold the other separators, so counting them sizes the vector exactly */
for (r = 0; cmd[r] != '\0'; r++) {
if (cmd[r] == '\t' || cmd[r] == '\r' || cmd[r] == '\n') {
cmd[r] = ' ';
}
if (cmd[r] == ' ') {
nsep++;
}
}
/* at most one argument per separator, plus the leading one and the NULL */
if (nsep > (size_t) INT_MAX - 1 || nsep > SIZE_MAX / sizeof(char *) - 2) {
return NULL;
}
capacity = nsep + 2;
argv = (char **) malloct(capacity * sizeof(char *));
if (argv == NULL) {
return NULL;
}
argv[argc++] = cmd;
for (r = 0, w = 0; cmd[r] != '\0';) {
if (quoted && cmd[r] == '\\' &&
(cmd[r + 1] == '\\' || cmd[r + 1] == '\"')) {
r++;
cmd[w++] = cmd[r++];
} else if (cmd[r] == '\"') {
quoted = !quoted;
cmd[w++] = cmd[r++];
} else if (cmd[r] == ' ' && !quoted) {
cmd[w++] = '\0';
assertf((size_t) argc < capacity - 1); /* the last slot holds the NULL */
argv[argc++] = cmd + w;
r++;
} else {
cmd[w++] = cmd[r++];
}
}
cmd[w] = '\0';
argv[argc] = NULL; /* callers may rely on argv[argc] == NULL */
*nargs = argc;
return argv;
}

View File

@@ -1,45 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: command line splitter, shared by the engine and */
/* htsserver */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSCMDLINE_DEFH
#define HTSCMDLINE_DEFH
#include "htsglobal.h"
/* Split "cmd" in place into a NULL-terminated argv vector of *nargs entries,
argv[0] being the program name and quotes left for the engine to strip.
Returns a malloct'ed vector of pointers into cmd (freet the vector, never its
entries), or NULL when it cannot be sized or allocated. */
char **hts_split_cmdline(char *cmd, int *nargs);
#endif

View File

@@ -447,22 +447,13 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
const char *fil) {
if (!*makeindex_done) {
if (*makeindex_fp) {
/* sized off link_escaped below: at the old flat 1024 a long first link
produced a redirect to a clipped URL */
char BIGSTK tempo[HTS_URLMAXSIZE * 2 + 64];
char BIGSTK tempo[1024];
if (makeindex_links == 1) {
char BIGSTK link_escaped[HTS_URLMAXSIZE * 2];
escape_uri_utf(makeindex_firstlink, link_escaped, sizeof(link_escaped));
/* no redirect beats one pointing at a clipped URL */
if (!sprintfbuff(
tempo,
"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">" CRLF,
link_escaped)) {
hts_log_print(opt, LOG_WARNING,
"index redirect omitted: first link too long (%s)",
makeindex_firstlink);
tempo[0] = '\0';
}
snprintf(tempo, sizeof(tempo),
"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">" CRLF,
link_escaped);
} else
tempo[0] = '\0';
hts_template_format(*makeindex_fp, template_footer,
@@ -705,19 +696,17 @@ int httpmirror(char *url1, httrackp * opt) {
// copier adresse(s) dans liste des adresses
{
char *a = url1;
const LLint list_sz = StringNotEmpty(opt->filelist)
? fsize_utf8(StringBuff(opt->filelist))
: 0;
/* two bytes reserved per list byte; -1 makes an undoublable size refused */
const LLint list_room =
list_sz > 0 ? (list_sz <= INT64_MAX / 2 ? list_sz * 2 : -1) : 0;
const size_t primary_len =
llint_grow_size_t(8192 + strlen(url1) * 2, list_room, 0);
int primary_len = 8192;
if (StringNotEmpty(opt->filelist)) {
primary_len += max(0, fsize_utf8(StringBuff(opt->filelist)) * 2);
}
primary_len += (int) strlen(url1) * 2;
// création de la première page, qui contient les liens de base à scanner
// c'est plus propre et plus logique que d'entrer à la main les liens dans la pile
// on bénéficie ainsi des vérifications et des tests du robot pour les liens "primaires"
primary = primary_len != (size_t) -1 ? (char *) malloct(primary_len) : NULL;
primary = (char *) malloct(primary_len);
if (!primary) {
printf("PANIC! : Not enough memory [%d]\n", __LINE__);
XH_extuninit;
@@ -898,7 +887,7 @@ int httpmirror(char *url1, httrackp * opt) {
}
if (filelist_buff != NULL) {
size_t filelist_ptr = 0;
int filelist_ptr = 0;
int n = 0;
char BIGSTK line[HTS_URLMAXSIZE * 2];

View File

@@ -145,7 +145,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
int argv_url = -1; // ==0 : utiliser cache et doit.log
char *argv_firsturl = NULL; // utilisé pour nommage par défaut
char *url = NULL; // URLS séparées par un espace
size_t url_sz = 65535;
int url_sz = 65535;
// the parametres
int httrack_logmode = 3; // ONE log file
@@ -224,22 +224,21 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
/* create x_argvblk buffer for transformed command line */
{
size_t current_size = 0;
const LLint size = fsize("config");
size_t blk_size;
int current_size = 0;
int size;
int na;
for(na = 0; na < argc; na++)
current_size += strlen(argv[na]) + 1;
/* a huge file named "config" must saturate, not wrap, the capacity */
blk_size = llint_grow_size_t(current_size, size > 0 ? size : 0, 32768);
x_argvblk = blk_size != (size_t) -1 ? (char *) malloct(blk_size) : NULL;
current_size += (int) (strlen(argv[na]) + 1);
if ((size = fsize("config")) > 0)
current_size += size;
x_argvblk = (char *) malloct(current_size + 32768);
if (x_argvblk == NULL) {
HTS_PANIC_PRINTF("Error, not enough memory");
htsmain_free();
return -1;
}
x_argvblk_size = blk_size;
x_argvblk_size = (size_t) (current_size + 32768);
x_argvblk[0] = '\0';
x_ptr = 0;
@@ -1457,29 +1456,20 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
FILE *fp = FOPEN(argv[na], "rb");
if (fp != NULL) {
size_t cl = strlen(url);
const size_t fzs = llint_to_size_t(fz);
const size_t capa = llint_grow_size_t(cl, fz, 8192);
int cl = (int) strlen(url);
if (capa == (size_t) -1) {
fclose(fp);
HTS_PANIC_PRINTF("File url list too large");
htsmain_free();
return -1;
}
ensureUrlCapacity(url, url_sz, capa);
ensureUrlCapacity(url, url_sz, cl + fz + 8192);
if (cl > 0) { /* don't stick! (3.43) */
url[cl] = ' ';
cl++;
}
if (fread(url + cl, 1, fzs, fp) != fzs) {
fclose(fp);
if (fread(url + cl, 1, fz, fp) != fz) {
HTS_PANIC_PRINTF("File url list could not be read");
htsmain_free();
return -1;
}
fclose(fp);
*(url + cl + fzs) = '\0';
*(url + cl + fz) = '\0';
}
}
}
@@ -2306,8 +2296,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
} else { // URL/filters
char catbuff[CATBUFF_SIZE];
const size_t urlSize = strlen(argv[na]);
const size_t capa = strlen(url) + urlSize + 32;
const int urlSize = (int) strlen(argv[na]);
const int capa = (int) (strlen(url) + urlSize + 32);
assertf(urlSize < HTS_URLMAXSIZE);
if (urlSize < HTS_URLMAXSIZE) {

View File

@@ -72,8 +72,7 @@ Please visit our Website: http://www.httrack.com
HTS_UNUSED: suppress unused-symbol warnings. HTS_STATIC: an unused-safe
static. HTS_PRINTF_FUN(fmt, arg): mark a printf-like function so the
compiler type-checks the format string at argument index fmt against the
varargs starting at arg. HTS_CHECK_RESULT: the return value carries the only
error signal, so dropping it is a bug; a (void) cast does not silence it. */
varargs starting at arg. */
#ifndef HTS_UNUSED
#ifdef __GNUC__
#define HTS_UNUSED __attribute__((unused))
@@ -81,13 +80,10 @@ Please visit our Website: http://www.httrack.com
#define HTS_STATIC static __attribute__((unused))
#define HTS_PRINTF_FUN(fmt, arg) __attribute__((format(printf, fmt, arg)))
#define HTS_CHECK_RESULT __attribute__((warn_unused_result))
#else
#define HTS_UNUSED
#define HTS_STATIC static
#define HTS_PRINTF_FUN(fmt, arg)
#define HTS_CHECK_RESULT
#endif
#endif
@@ -340,20 +336,16 @@ typedef int INTsys;
#endif
/* Socket-handle type. An unsigned integer wide enough for a Windows SOCKET;
a plain int file descriptor on POSIX. T_SOCP is its printf conversion,
'%' included: unsigned __int64 on Win64 must not be printed with "%d". */
a plain int file descriptor on POSIX. */
#ifdef _WIN32
#if defined(_WIN64)
typedef unsigned __int64 T_SOC;
#define T_SOCP "%" PRIu64
#else
typedef unsigned __int32 T_SOC;
#define T_SOCP "%" PRIu32
#endif
#else
typedef int T_SOC;
#define T_SOCP "%d"
#endif
/* Buffer size for a printed network address (IPv4 or IPv6, NUL included). */

View File

@@ -124,9 +124,7 @@ typedef struct help_wizard_buffers {
char stropt[2048]; // options
char stropt2[2048]; // options longues
char strwild[2048]; // wildcards
/* holds all four of the above plus separators: at 4096 a long answer set
clipped the filters off the command line */
char cmd[HTS_URLMAXSIZE * 2 + 3 * 2048 + 4];
char cmd[4096];
char str[256];
char *argv[256];
} help_wizard_buffers;
@@ -158,7 +156,9 @@ void help_wizard(httrackp * opt) {
char *a;
//
if (buffers == NULL) {
if (urls == NULL || mainpath == NULL || projname == NULL || stropt == NULL
|| stropt2 == NULL || strwild == NULL || cmd == NULL || str == NULL
|| argv == NULL) {
fprintf(stderr, "* memory exhausted in %s, line %d\n", __FILE__, __LINE__);
return;
}
@@ -251,7 +251,6 @@ void help_wizard(httrackp * opt) {
strcatbuff(stropt2, "--update ");
break;
case 0:
freet(buffers);
return;
break;
}
@@ -310,23 +309,14 @@ void help_wizard(httrackp * opt) {
printf("\n");
if (strlen(stropt) == 1)
stropt[0] = '\0'; // aucune
/* the tail is the filter list, and cmd is split into the argv handed to
hts_main() below: a clipped line would silently widen the crawl */
if (!sprintfbuff(cmd, "%s %s %s %s", urls, stropt, stropt2, strwild)) {
printf("* command line too long (%d bytes max)\n",
(int) sizeof(cmd) - 1);
freet(buffers);
return;
}
snprintf(cmd, sizeof(cmd), "%s %s %s %s", urls, stropt, stropt2, strwild);
printf("---> Wizard command line: httrack %s\n\n", cmd);
printf("Ready to launch the mirror? (Y/n) :");
fflush(stdout);
linput(stdin, str, 250);
if (strnotempty(str)) {
if (!((str[0] == 'y') || (str[0] == 'Y'))) {
freet(buffers);
if (!((str[0] == 'y') || (str[0] == 'Y')))
return;
}
}
printf("\n");
@@ -350,7 +340,7 @@ void help_wizard(httrackp * opt) {
}
/* Free buffers */
freet(buffers);
free(buffers);
#undef urls
#undef mainpath
#undef projname

View File

@@ -704,16 +704,18 @@ T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
/* Check for errors */
if (soc == INVALID_SOCKET) {
if (retour) {
if (!strnotempty(retour->msg)) {
if (retour->msg) {
if (!strnotempty(retour->msg)) {
#ifdef _WIN32
int last_errno = WSAGetLastError();
int last_errno = WSAGetLastError();
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
#else
int last_errno = errno;
int last_errno = errno;
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
#endif
}
}
}
}
@@ -2205,7 +2207,7 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
#if DEBUG
printf("erreur gethostbyname\n");
#endif
if (retour != NULL) {
if (retour && retour->msg) {
#ifdef _WIN32
snprintf(retour->msg, sizeof(retour->msg),
"Unable to get server's address: %s", error);
@@ -2236,7 +2238,7 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
DEBUG_W("socket()=%d\n" _(int) soc);
#endif
if (soc == INVALID_SOCKET) {
if (retour != NULL) {
if (retour && retour->msg) {
#ifdef _WIN32
int last_errno = WSAGetLastError();
@@ -2260,8 +2262,17 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
&bind_addr, &error) == NULL
|| bind(soc, &SOCaddr_sockaddr(bind_addr),
SOCaddr_size(bind_addr)) != 0) {
snprintf(retour->msg, sizeof(retour->msg),
"Unable to bind the specificied server address: %s", error);
if (retour && retour->msg) {
#ifdef _WIN32
snprintf(retour->msg, sizeof(retour->msg),
"Unable to bind the specificied server address: %s",
error);
#else
snprintf(retour->msg, sizeof(retour->msg),
"Unable to bind the specificied server address: %s",
error);
#endif
}
deletesoc(soc);
return INVALID_SOCKET;
}
@@ -2308,7 +2319,7 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
#if HDEBUG
printf("unable to connect!\n");
#endif
if (retour != NULL) {
if (retour != NULL && retour->msg) {
#ifdef _WIN32
const int last_errno = WSAGetLastError();
@@ -2587,15 +2598,13 @@ void deletesoc(T_SOC soc) {
if (closesocket(soc) != 0) {
int err = WSAGetLastError();
fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
strerror(err));
fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err));
}
#else
if (close(soc) != 0) {
const int err = errno;
fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
strerror(err));
fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err));
}
#endif
#if HTS_WIDE_DEBUG
@@ -3008,7 +3017,7 @@ int finput(T_SOC fd, char *s, int max) {
}
// Like linput, but in memory (optimized)
int binput(const char *buff, char *s, int max) {
int binput(char *buff, char *s, int max) {
int count = 0;
int destCount = 0;

View File

@@ -262,7 +262,7 @@ HTS_INLINE void time_rfc822_local(char *s, struct tm *A);
HTS_INLINE int sendc(htsblk * r, const char *s);
int finput(T_SOC fd, char *s, int max);
int binput(const char *buff, char *s, int max);
int binput(char *buff, char *s, int max);
int linput(FILE * fp, char *s, int max);
int linputsoc(T_SOC soc, char *s, int max);
int linputsoc_t(T_SOC soc, char *s, int max, int timeout);
@@ -607,21 +607,6 @@ static HTS_UNUSED size_t llint_to_size_t(LLint o) {
}
}
/* Capacity for @p used bytes plus @p extra more plus @p slack spare;
(size_t) -1 if the total exceeds (size_t) -2 or @p extra is negative
(llint_to_size_t() would map that to a huge valid-looking size). */
static HTS_UNUSED size_t llint_grow_size_t(size_t used, LLint extra,
size_t slack) {
const size_t max = (size_t) -2; /* (size_t) -1 is the error value */
const size_t e = extra >= 0 ? llint_to_size_t(extra) : (size_t) -1;
if (e == (size_t) -1 || used > max || slack > max - used ||
e > max - used - slack) {
return (size_t) -1;
}
return used + e + slack;
}
/* dirent() compatibility */
#ifdef _WIN32
/* Holds a UTF-8 d_name: MAX_PATH (260) UTF-16 units expand to <=3 bytes each.

View File

@@ -33,7 +33,6 @@ Please visit our Website: http://www.httrack.com
#ifndef HTSSAFE_DEFH
#define HTSSAFE_DEFH
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -460,37 +459,6 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
return b->buf;
}
/**
* Formatted print into dest (capacity size, NUL included), truncating to fit
* and always NUL-terminating. Returns HTS_TRUE if the whole output fit; the
* result is the only truncation signal, so it must be acted on. Unlike
* strcpybuff() it never aborts, so it suits text built from remote input.
*/
static HTS_INLINE HTS_UNUSED HTS_CHECK_RESULT HTS_PRINTF_FUN(3, 4) hts_boolean
slprintfbuff(char *dest, size_t size, const char *fmt, ...) {
va_list args;
int ret;
assertf(dest != NULL && size != 0);
va_start(args, fmt);
ret = vsnprintf(dest, size, fmt, args);
va_end(args);
/* pre-C99 runtimes (msvcrt _vsnprintf) return -1 and do not terminate */
dest[size - 1] = '\0';
return ret >= 0 && (size_t) ret < size ? HTS_TRUE : HTS_FALSE;
}
/**
* slprintfbuff() over the in-scope array ARR (capacity = sizeof(ARR)).
* On GCC/Clang a pointer is a compile error; use slprintfbuff() for those.
*/
#if (defined(__GNUC__) && !defined(__cplusplus))
#define sprintfbuff(ARR, ...) \
slprintfbuff((ARR), sizeof(ARR) + htsbuff_must_be_array_(ARR), __VA_ARGS__)
#else
#define sprintfbuff(ARR, ...) slprintfbuff((ARR), sizeof(ARR), __VA_ARGS__)
#endif
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
added bounds checking. freet() also NULLs the freed pointer and tolerates
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the

View File

@@ -51,7 +51,6 @@ Please visit our Website: http://www.httrack.com
#include "htscache_selftest.h"
#include "htsdns_selftest.h"
#include "htscharset.h"
#include "htscmdline.h"
#include "htsencoding.h"
#include "htsftp.h"
#include "htsmd5.h"
@@ -569,54 +568,6 @@ static int string_safety_selftests(void) {
return 1;
}
/* sprintfbuff: truncate-and-report. Must never abort (its callers format
remote banners) nor write past the array, which the canary catches. */
{
struct {
char dst[8];
char canary[8];
} s;
const char *const big = "0123456789abcdefghijklmnopqrstuvwxyz";
/* repoison before every call, or an implementation that measures first and
writes nothing "passes" the truncating cases on the previous content */
#define POISON_DST() memset(s.dst, '#', sizeof(s.dst))
memset(&s, '#', sizeof(s));
if (!sprintfbuff(s.dst, "%s-%d", "ab", 42) || strcmp(s.dst, "ab-42") != 0)
return 1;
/* exact fit: 7 characters plus the NUL */
POISON_DST();
if (!sprintfbuff(s.dst, "%s", "1234567") || strcmp(s.dst, "1234567") != 0)
return 1;
/* one over, then far over: truncated to the prefix, terminated, reported */
POISON_DST();
if (sprintfbuff(s.dst, "%s", "12345678") || strcmp(s.dst, "1234567") != 0)
return 1;
POISON_DST();
if (sprintfbuff(s.dst, "%s", big) || strcmp(s.dst, "0123456") != 0)
return 1;
/* explicit-capacity form, down to the degenerate size 1 */
{
char *const p = s.dst;
POISON_DST();
if (slprintfbuff(p, 1, "%s", "x") || p[0] != '\0')
return 1;
POISON_DST();
if (!slprintfbuff(p, sizeof(s.dst), "%s", "ok") || strcmp(p, "ok") != 0)
return 1;
}
#undef POISON_DST
if (memcmp(s.canary, "########", sizeof(s.canary)) != 0)
return 1;
}
/* StringCatN/StringSetLength must eval SIZE once: (n_eval++, V) leaves
n_eval == 2 on a double-eval macro. */
{
@@ -1175,128 +1126,6 @@ static int st_unescape_bounds(httrackp *opt, int argc, char **argv) {
return 0;
}
// hts_split_cmdline(): the vector must grow with the argument count, and a
// quote inside a value must not end the argument and hand -V to the parser.
static int st_cmdlinesplit(httrackp *opt, int argc, char **argv) {
char line[512];
char **args;
int nargs = 0;
(void) opt;
(void) argc;
(void) argv;
// control: every separator splits, and argv[0] is the program name
strcpybuff(line, "httrack http://x/ --quiet\t-c8\n-O out");
args = hts_split_cmdline(line, &nargs);
assertf(args != NULL && nargs == 6);
assertf(args[nargs] == NULL); // callers may walk to the terminator
assertf(strcmp(args[0], "httrack") == 0);
assertf(strcmp(args[1], "http://x/") == 0);
assertf(strcmp(args[2], "--quiet") == 0);
assertf(strcmp(args[3], "-c8") == 0);
assertf(strcmp(args[4], "-O") == 0);
assertf(strcmp(args[5], "out") == 0);
freet(args);
// the template pads with whitespace: empty arguments are kept (the engine
// skips them), so the count is one per separator
strcpybuff(line, "httrack --quiet");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 3 && args[1][0] == '\0');
assertf(strcmp(args[2], "--quiet") == 0);
freet(args);
// a quoted run keeps both its spaces and its quotes: the engine unquotes
strcpybuff(line, "httrack --user-agent \"Mozilla 5.0\" -c8");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 4);
assertf(strcmp(args[2], "\"Mozilla 5.0\"") == 0);
assertf(strcmp(args[3], "-c8") == 0);
freet(args);
// an escaped quote is a literal quote, not the end of the argument: the
// engine strips only the outer pair
strcpybuff(line, "httrack --user-agent \"x\\\" -V \\\"touch /tmp/pwn\" -c8");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 4);
assertf(strcmp(args[2], "\"x\" -V \"touch /tmp/pwn\"") == 0);
assertf(strcmp(args[3], "-c8") == 0);
freet(args);
// \\ is a literal backslash, so a Windows path survives
strcpybuff(line, "httrack --path \"C:\\\\dir\\\\sub\"");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 3);
assertf(strcmp(args[2], "\"C:\\dir\\sub\"") == 0);
freet(args);
// outside a quoted run a backslash is literal: the url and wildcard-filter
// fields, which the wizard cannot quote, read as before
strcpybuff(line, "httrack -*\\** +*.png");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 3);
assertf(strcmp(args[1], "-*\\**") == 0);
assertf(strcmp(args[2], "+*.png") == 0);
freet(args);
// a quoted run leaves slots unused, so the terminator has to be written and
// not inherited: size the vector from a full line first, so freeing it hands
// the same chunk back with stale pointers in those slots
strcpybuff(line, "httrack a b c d e");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 6);
freet(args);
strcpybuff(line, "httrack \"a b c d e\"");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 2);
assertf(args[nargs] == NULL);
freet(args);
// an unterminated quote protects the rest of the line, as one argument
strcpybuff(line, "httrack --footer \"unbalanced -V x");
args = hts_split_cmdline(line, &nargs);
assertf(nargs == 3);
assertf(strcmp(args[2], "\"unbalanced -V x") == 0);
freet(args);
// past the 1024 entries the vector used to hold: distinct arguments, so a
// write beyond the allocation cannot read back as the expected parse
{
const int n = 2000;
const size_t size = 16 * (size_t) n + 16;
char *big = malloct(size);
size_t pos = 0;
int i;
assertf(big != NULL);
pos = (size_t) snprintf(big, size, "httrack");
assertf(pos < size);
for (i = 0; i < n; i++) {
// snprintf returns what it wanted to write, so accumulating it blind
// would let the next size argument wrap
const int len = snprintf(big + pos, size - pos, " a%d", i);
assertf(len > 0 && (size_t) len < size - pos);
pos += (size_t) len;
}
args = hts_split_cmdline(big, &nargs);
assertf(args != NULL && nargs == n + 1);
assertf(args[nargs] == NULL);
for (i = 0; i < n; i++) {
char expect[16];
snprintf(expect, sizeof(expect), "a%d", i);
assertf(strcmp(args[i + 1], expect) == 0);
}
freet(args);
freet(big);
}
printf("cmdline-split self-test OK\n");
return 0;
}
static int st_hashtable(httrackp *opt, int argc, char **argv) {
char *snum;
unsigned long count = 0;
@@ -2194,77 +2023,6 @@ static int st_fsize(httrackp *opt, int argc, char **argv) {
return rc;
}
/* 4GB+100KB wraps to ~108KB through an int, and needs 33 unsigned bits. A
macro, not a static const: MSVC's C mode (/TC) rejects a const object
used inside another object's static initializer below (C2099). */
#define HTS_ST_GROWSIZE_OVER32 (4LL * 1024 * 1024 * 1024 + 100 * 1024)
/* llint_grow_size_t() sizes the buffer holding a whole -%S list file: the
result must be the exact 64-bit sum or a clean refusal, never a short one. */
static int st_growsize(httrackp *opt, int argc, char **argv) {
enum { REFUSE, ACCEPT, WIDTH };
static const struct {
size_t used;
LLint extra;
size_t slack;
int want;
} cases[] = {
{0, 0, 0, ACCEPT},
{10, 100, 8192, ACCEPT},
{(size_t) -2 - 8, 4, 4, ACCEPT}, /* exact fit, no room to spare */
{(size_t) -2, 0, 0, ACCEPT}, /* largest representable capacity */
{0, -1, 0, REFUSE}, /* fsize() failure */
/* -1 already maps to SIZE_MAX; only this exercises the negative guard */
{0, -4096, 0, REFUSE},
{(size_t) -1, 1, 0, REFUSE},
{(size_t) -2, 0, 1, REFUSE}, /* slack alone overruns */
{(size_t) -1 - 8, 4, 4, REFUSE}, /* total would be the error value */
{(size_t) -1 - 8, 4, 8, REFUSE},
{0, HTS_ST_GROWSIZE_OVER32, 8192,
WIDTH}, /* 32-bit size_t can't hold these */
{10, HTS_ST_GROWSIZE_OVER32, 8192, WIDTH},
};
size_t k;
int rc = 0;
(void) opt;
(void) argc;
(void) argv;
for (k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
const size_t used = cases[k].used, slack = cases[k].slack;
const LLint extra = cases[k].extra;
const size_t got = llint_grow_size_t(used, extra, slack);
const hts_boolean refused = got == (size_t) -1 ? HTS_TRUE : HTS_FALSE;
const hts_boolean exact =
!refused && extra >= 0 && got - used - slack == (size_t) extra;
hts_boolean ok;
switch (cases[k].want) {
case ACCEPT:
ok = exact;
break;
case REFUSE:
ok = refused;
break;
default:
ok = sizeof(size_t) >= sizeof(LLint) ? exact : refused;
break;
}
if (!ok) {
fprintf(stderr,
"growsize: grow(" LLintP ", " LLintP ", " LLintP ") = " LLintP
" (want %s)\n",
(LLint) used, extra, (LLint) slack, (LLint) got,
cases[k].want == REFUSE ? "refusal" : "exact sum");
rc = 1;
}
}
printf("growsize self-test %s\n", rc == 0 ? "OK" : "FAILED");
return rc;
}
static int st_savename(httrackp *opt, int argc, char **argv) {
lien_adrfilsave afs;
cache_back cache;
@@ -2659,20 +2417,17 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
static t_cookie ck2;
htsblk r;
char host[600];
char line[64]; /* treathead NUL-cuts the header in place: never a literal */
memset(&r, 0, sizeof(r));
memset(host, 'a', sizeof(host) - 1);
host[sizeof(host) - 1] = '\0';
ck2.max_len = (int) sizeof(ck2.data);
ck2.data[0] = '\0';
strcpybuff(line, "Set-Cookie: SID=1; path=/");
treathead(&ck2, host, "/", &r, line);
treathead(&ck2, host, "/", &r, "Set-Cookie: SID=1; path=/");
if (strnotempty(ck2.data)) // oversize-host cookie was not dropped
err = 1;
/* control: a normal host still yields a cookie through treathead */
strcpybuff(line, "Set-Cookie: SID=1; path=/");
treathead(&ck2, dom, "/", &r, line);
treathead(&ck2, dom, "/", &r, "Set-Cookie: SID=1; path=/");
if (strstr(ck2.data, "SID") == NULL) // guard wrongly dropped a valid cookie
err = 1;
}
@@ -2933,32 +2688,6 @@ static int st_makeindex(httrackp *opt, int argc, char **argv) {
assertf(strstr(buf, "Refresh") != NULL);
assertf(strstr(buf, "example.com") != NULL);
/* a first link whose escaped form overruns the old flat 1024-byte tempo: the
redirect must carry the whole URL, not a clipped prefix */
{
char BIGSTK link[HTS_URLMAXSIZE * 2];
char *p = link;
strcpybuff(link, "http://example.com/");
p += strlen(link);
memset(p, 'a', 1200);
p += 1200;
strcpy(p, "/end.html");
done = 0;
fp = fopen(path, "wb");
assertf(fp != NULL);
hts_finish_makeindex(opt, &done, &fp, 1, link, "%s%s", "", "");
assertf(fp == NULL);
fp = fopen(path, "rb");
assertf(fp != NULL);
n = fread(buf, 1, sizeof(buf) - 1, fp);
fclose(fp);
buf[n] = '\0';
/* the closing quote proves the URL was not clipped mid-way */
assertf(strstr(buf, "/end.html\">") != NULL);
}
/* no single link: footer only, no refresh meta */
done = 0;
fp = fopen(path, "wb");
@@ -3188,7 +2917,7 @@ static int ae_write_packed(const char *path, int windowBits,
deflateEnd(&strm);
return 1;
}
strm.next_in = (const Bytef *) src;
strm.next_in = (Bytef *) src;
strm.avail_in = (uInt) len;
do {
size_t n;
@@ -5107,9 +4836,6 @@ static const struct selftest_entry {
st_footerfmt},
{"unescape-bounds", "", "unescapers reserve the NUL byte (no 1-byte OOB)",
st_unescape_bounds},
{"cmdline-split", "",
"webhttrack command-line to argv split (bounds, quoting)",
st_cmdlinesplit},
{"hashtable", "<count|file>", "coucal hashtable stress test", st_hashtable},
{"strsafe", "[overflow|overflow-buff|overflow-src [str]]",
"bounded string-op self-test", st_strsafe},
@@ -5142,8 +4868,6 @@ static const struct selftest_entry {
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
st_sniff},
{"fsize", "<dir>", "file size past the 2GB signed-32-bit wrap", st_fsize},
{"growsize", "", "buffer capacity for a 64-bit file size (no int wrap)",
st_growsize},
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
{"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
st_cacheindex},

View File

@@ -147,8 +147,7 @@ HTS_UNUSED static int LANG_LIST(const char *path, char *buffer, size_t size);
// 0- Init the URL catcher with standard port
// smallserver_init(&port,&return_host);
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort,
const char *bindAddr) {
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort) {
T_SOC soc;
if (defaultPort <= 0) {
@@ -161,12 +160,12 @@ T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort,
int i = 0;
do {
soc = smallserver_init(&try_to_listen_to[i], adr_prox, bindAddr);
soc = smallserver_init(&try_to_listen_to[i], adr_prox);
*port_prox = try_to_listen_to[i];
i++;
} while((soc == INVALID_SOCKET) && (try_to_listen_to[i] >= 0));
} else {
soc = smallserver_init(&defaultPort, adr_prox, bindAddr);
soc = smallserver_init(&defaultPort, adr_prox);
*port_prox = defaultPort;
}
return soc;
@@ -244,10 +243,9 @@ static int my_gethostname(char *h_loc, size_t size) {
}
// smallserver_init(&port,&return_host);
T_SOC smallserver_init(int *port, char *adr, const char *bindAddr) {
T_SOC smallserver_init(int *port, char *adr) {
T_SOC soc = INVALID_SOCKET;
char h_loc[256 + 2];
SOCaddr server;
commandRunning = commandEnd = commandReturn = commandReturnSet =
commandEndRequested = 0;
@@ -258,23 +256,25 @@ T_SOC smallserver_init(int *port, char *adr, const char *bindAddr) {
free(commandReturnCmdl);
commandReturnCmdl = NULL;
SOCaddr_initany(server);
if (bindAddr != NULL && *bindAddr != '\0') {
/* advertise the bound address, else the URL we print is unreachable */
if (strlen(bindAddr) >= sizeof(h_loc) || !gethost(bindAddr, &server)) {
return INVALID_SOCKET;
}
strcpybuff(h_loc, bindAddr);
} else if (my_gethostname(h_loc, 256) != 0) { // host name
return INVALID_SOCKET;
}
if (my_gethostname(h_loc, 256) == 0) { // host name
SOCaddr server;
if ((soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0)) !=
INVALID_SOCKET) {
SOCaddr_initport(server, *port);
if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
if (listen(soc, 10) >= 0) {
strcpy(adr, h_loc);
SOCaddr_initany(server);
if ((soc =
(T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM,
0)) != INVALID_SOCKET) {
SOCaddr_initport(server, *port);
if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
if (listen(soc, 10) >= 0) {
strcpy(adr, h_loc);
} else {
#ifdef _WIN32
closesocket(soc);
#else
close(soc);
#endif
soc = INVALID_SOCKET;
}
} else {
#ifdef _WIN32
closesocket(soc);
@@ -283,13 +283,6 @@ T_SOC smallserver_init(int *port, char *adr, const char *bindAddr) {
#endif
soc = INVALID_SOCKET;
}
} else {
#ifdef _WIN32
closesocket(soc);
#else
close(soc);
#endif
soc = INVALID_SOCKET;
}
}
return soc;
@@ -318,104 +311,6 @@ typedef struct {
error_redirect = "/server/error.html"; \
} while(0)
/* Longest "sid" value worth unescaping: the expected one is an md5 hex digest,
so anything near this is already invalid and is rejected unread. */
#define SID_VALUE_MAX 64
/** Does the urlencoded request body present the expected session id?
True only if at least one "sid" field is present and every occurrence
matches, so it holds whichever one a later last-write-wins parse keeps.
Non-destructive: it runs before the body is tokenized in place. */
static hts_boolean body_sid_is_valid(const char *body, const char *expected) {
const char *s = body;
hts_boolean seen = HTS_FALSE;
while (s != NULL && *s != '\0') {
const char *const amp = strchr(s, '&');
const char *const eq = strchr(s, '=');
if (eq != NULL && (amp == NULL || eq < amp) && (size_t) (eq - s) == 3 &&
strncmp(s, "sid", 3) == 0) {
const size_t len = amp != NULL ? (size_t) (amp - eq - 1) : strlen(eq + 1);
hts_boolean match = HTS_FALSE;
if (len < SID_VALUE_MAX) {
char raw[SID_VALUE_MAX];
String value = STRING_EMPTY;
memcpy(raw, eq + 1, len);
raw[len] = '\0';
unescapehttp(raw, &value);
/* StringBuff is NULL until written, so an empty value lands here. */
if (StringBuff(value) != NULL &&
strcmp(StringBuff(value), expected) == 0) {
match = HTS_TRUE;
}
StringFree(value);
}
if (!match) {
return HTS_FALSE;
}
seen = HTS_TRUE;
}
s = amp != NULL ? amp + 1 : NULL;
}
return seen;
}
/** Append src to the NUL-terminated dst of capacity size (NUL included).
False, leaving dst untouched, if it would not fit: unlike strcatbuff() this
never aborts, because every piece appended here is client-supplied. */
static hts_boolean path_append(char *dst, size_t size, const char *src) {
const size_t used = strlen(dst);
const size_t len = strlen(src);
/* dst holds at most size-1 bytes, so "size - used" is >= 1 and the untrusted
len stays alone: "used + len < size" could wrap and pass. */
if (len >= size - used) {
return HTS_FALSE;
}
memcpy(dst + used, src, len + 1);
return HTS_TRUE;
}
/* Append c to dst as an HTML entity, or return HTS_FALSE if it needs none. */
static hts_boolean cat_html_escaped(String *dst, char c) {
switch (c) {
case '<':
StringCat(*dst, "&lt;");
break;
case '>':
StringCat(*dst, "&gt;");
break;
case '&':
StringCat(*dst, "&amp;");
break;
case '\'':
StringCat(*dst, "&#39;");
break;
default:
return HTS_FALSE;
}
return HTS_TRUE;
}
/* Append the value of a double-quoted command-line argument: escaped for HTML,
which the browser undoes when it posts the command line back, and for the
argv splitter, which does not. */
static void cat_cmdline_arg(String *output, const char *value) {
const char *a;
for (a = value; *a != '\0'; a++) {
if (*a == '\\' || *a == '\"') {
StringCat(*output, "\\");
}
if (!cat_html_escaped(output, *a)) {
StringMemcat(*output, a, 1);
}
}
}
int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
int timeout = 30;
int retour = 0;
@@ -427,9 +322,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
String tmpbuff = STRING_EMPTY;
String tmpbuff2 = STRING_EMPTY;
String fspath = STRING_EMPTY;
/* Project directory this server set up; the only root /website/ serves from,
and deliberately not cleared between requests. */
String website = STRING_EMPTY;
char catbuff[CATBUFF_SIZE];
/* Load strings */
@@ -503,7 +395,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
T_SOC soc_c;
LLint length = 0;
const char *error_redirect = NULL;
hts_boolean denied = HTS_FALSE;
line[0] = '\0';
buffer[0] = '\0';
@@ -615,22 +506,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
}
/* Authenticate the body before parsing it: every field it carries is
written straight into the global key store below, "command" included,
and that one reaches the engine. Checking afterwards cannot work — the
damage is already done, and the pre-seeded "sid" above would compare
equal to itself for a request that simply omits the field. */
if (meth && buffer[0]) {
intptr_t expected = 0;
if (!coucal_readptr(NewLangList, "_sid", &expected) ||
!body_sid_is_valid(buffer, (const char *) expected)) {
buffer[0] = '\0';
meth = 0;
denied = HTS_TRUE;
}
}
/* check variables */
if (meth && buffer[0]) {
char *s = buffer;
@@ -651,6 +526,20 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
}
/* Error check */
{
intptr_t adr = 0;
intptr_t adr2 = 0;
if (coucal_readptr(NewLangList, "sid", &adr)) {
if (coucal_readptr(NewLangList, "_sid", &adr2)) {
if (strcmp((char *) adr, (char *) adr2) != 0) {
meth = 0;
}
}
}
}
/* Check variables (internal) */
if (meth) {
int doLoad = 0;
@@ -841,11 +730,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
if (!structcheck(StringBuff(tmpbuff))) {
FILE *fp;
/* Both halves of fspath come from posted fields, so a ".."
in them would escape the mirror once served. */
if (strstr(StringBuff(fspath), "..") == NULL) {
StringCopy(website, StringBuff(fspath));
}
StringCat(tmpbuff, "winprofile.ini");
fp = fopen(StringBuff(tmpbuff), "wb");
if (fp != NULL) {
@@ -918,7 +802,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
/* Response */
if (meth) {
hts_boolean virtualpath = HTS_FALSE;
int virtualpath = 0;
char *pos;
char *url = strchr(line1, ' ');
@@ -929,11 +813,11 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
char *qpos;
/* get the URL */
fsfile[0] = '\0';
if (error_redirect == NULL) {
if ((qpos = strchr(url, '?'))) {
*qpos = '\0';
}
fsfile[0] = '\0';
if (strcmp(url, "/") == 0) {
file = "/server/index.html";
meth = 2;
@@ -946,7 +830,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
if (strncmp(file, "/website/", 9) == 0) {
virtualpath = HTS_TRUE;
virtualpath = 1;
}
/* override */
@@ -960,26 +844,18 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
}
/* the override above may have swapped a mirror path for a GUI page */
virtualpath = strncmp(file, "/website/", 9) == 0;
if (strlen(path) + strlen(file) + 32 < sizeof(fsfile)) {
if (strncmp(file, "/website/", 9) != 0) {
sprintf(fsfile, "%shtml%s", path, file);
} else {
intptr_t adr = 0;
if (!virtualpath) {
if (!path_append(fsfile, sizeof(fsfile), path) ||
!path_append(fsfile, sizeof(fsfile), "html") ||
!path_append(fsfile, sizeof(fsfile), file)) {
fsfile[0] = '\0';
}
} else if (StringNotEmpty(website)) {
/* Never the posted "projpath": a client root reads any file. */
if (!path_append(fsfile, sizeof(fsfile), StringBuff(website)) ||
!path_append(fsfile, sizeof(fsfile), "/") ||
!path_append(fsfile, sizeof(fsfile), file + 9)) {
fsfile[0] = '\0';
if (coucal_readptr(NewLangList, "projpath", &adr)) {
sprintf(fsfile, "%s%s", (char *) adr, file + 9);
}
}
}
/* path itself may hold ".." (webhttrack passes "<bin>/../share"), so
only the untrusted halves are checked: file here, website above. */
if (fsfile[0] && strstr(file, "..") == NULL
&& (fp = fopen(fsfile, "rb"))) {
char ok[] =
@@ -1027,16 +903,16 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
}
StringMemcat(headers, redir, strlen(redir));
/* client-supplied: a CR/LF here would split the response */
if (newfile[strcspn(newfile, "\r\n")] == '\0') {
StringCat(headers, "Location: ");
StringCat(headers, newfile);
StringCat(headers, "\r\n");
{
char tmp[256];
if (strlen(file) < sizeof(tmp) - 32) {
sprintf(tmp, "Location: %s\r\n", newfile);
StringMemcat(headers, tmp, strlen(tmp));
}
}
coucal_write(NewLangList, "redirect", (intptr_t) NULL);
} else if (!virtualpath && is_html(file)) {
/* GUI templates only: ${_sid} in a mirrored page would hand the
crawled site the session id that authenticates commands */
} else if (is_html(file)) {
int outputmode = 0;
StringMemcat(headers, ok, sizeof(ok) - 1);
@@ -1064,7 +940,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
int p;
int format = 0;
int listDefault = 0;
hts_boolean unquoted = HTS_FALSE;
name[0] = '\0';
strlncatbuff(name, str, sizeof(name_), n);
@@ -1074,12 +949,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
} else if ((p = strfield(name, "html:"))) {
name += p;
format = 1;
} else if ((p = strfield(name, "unquoted:"))) {
name += p;
unquoted = HTS_TRUE;
} else if ((p = strfield(name, "arg:"))) {
name += p;
format = 5;
} else if ((p = strfield(name, "list:"))) {
name += p;
format = 2;
@@ -1216,8 +1085,8 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
test:<if ==0>:<if ==1>:<if == 2>..
ztest:<if == 0 || !exist>:<if == 1>:<if == 2>..
*/
else if ((p = strfield(name, "test:")) ||
(p = strfield(name, "ztest:"))) {
else if ((p = strfield(name, "test:"))
|| (p = strfield(name, "ztest:"))) {
intptr_t adr = 0;
char *pos2;
int ztest = (name[0] == 'z');
@@ -1320,12 +1189,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
}
}
/* consumed here: it shares nothing with the list and
option formats below */
if (format == 5 && langstr != NULL && outputmode != -1) {
cat_cmdline_arg(&output, langstr);
langstr = NULL;
}
if (langstr && outputmode != -1) {
switch (format) {
case 0:
@@ -1343,18 +1206,18 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
StringMemcat(output, &c, 1);
}
a += 2;
} else if (unquoted && a[0] == '\"') {
/* the browser posts an entity back as a raw
quote, which would open a quoted run in the
argv splitter; a URI cannot hold one anyway */
StringCat(output, "%22");
} else if (outputmode &&
cat_html_escaped(&output, a[0])) {
/* appended as an entity */
} else if (outputmode && a[0] == '<') {
StringCat(output, "&lt;");
} else if (outputmode && a[0] == '>') {
StringCat(output, "&gt;");
} else if (outputmode && a[0] == '&') {
StringCat(output, "&amp;");
} else if (outputmode && a[0] == '\'') {
StringCat(output, "&#39;");
} else if (outputmode == 3 && a[0] == ' ') {
StringCat(output, "%20");
} else if (outputmode >= 2 &&
((unsigned char) a[0]) < 32) {
} else if (outputmode >= 2
&& ((unsigned char) a[0]) < 32) {
char tmp[32];
sprintf(tmp, "%%%02x", (unsigned char) a[0]);
@@ -1415,10 +1278,20 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
StringClear(tmpbuff);
break;
case '<':
StringCat(tmpbuff, "&lt;");
break;
case '>':
StringCat(tmpbuff, "&gt;");
break;
case '&':
StringCat(tmpbuff, "&amp;");
break;
case '\'':
StringCat(tmpbuff, "&#39;");
break;
default:
if (!cat_html_escaped(&tmpbuff, *fstr)) {
StringMemcat(tmpbuff, fstr, 1);
}
StringMemcat(tmpbuff, fstr, 1);
break;
}
fstr++;
@@ -1458,9 +1331,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
}
#endif
} else {
if (is_html(file)) {
StringMemcat(headers, ok, sizeof(ok) - 1);
} else if (is_text(file)) {
if (is_text(file)) {
StringMemcat(headers, ok_text, sizeof(ok_text) - 1);
} else if (is_js(file)) {
StringMemcat(headers, ok_js, sizeof(ok_js) - 1);
@@ -1497,11 +1368,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
StringCat(output, error);
}
}
} else if (denied) {
StringCat(headers, "HTTP/1.0 403 Forbidden\r\n"
"Server: httrack small server\r\n"
"Content-type: text/html\r\n");
StringCat(output, "Missing or invalid session id.\r\n");
} else {
#ifdef _DEBUG
char error_hdr[] =
@@ -1567,7 +1433,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
StringFree(tmpbuff);
StringFree(tmpbuff2);
StringFree(fspath);
StringFree(website);
if (buffer)
free(buffer);

View File

@@ -43,11 +43,8 @@ Please visit our Website: http://www.httrack.com
// Fonctions
void socinput(T_SOC soc, char *s, int max);
/* Listen on bindAddr, or every interface if NULL/empty; adr (>= 258 bytes) gets
the address to advertise. INVALID_SOCKET on error. */
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort,
const char *bindAddr);
T_SOC smallserver_init(int *port, char *adr, const char *bindAddr);
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort);
T_SOC smallserver_init(int *port, char *adr);
int smallserver(T_SOC soc, char *url, char *method, char *data, char *path);
#define CATCH_RESPONSE \

View File

@@ -1318,19 +1318,19 @@ HTSEXT_API hts_boolean hts_findnext(find_handle find) {
if (find) {
#ifdef _WIN32
if ((FindNextFileA(find->handle, &find->hdata)))
return HTS_TRUE;
return 1;
#else
char catbuff[CATBUFF_SIZE];
memset(&(find->filestat), 0, sizeof(find->filestat));
if ((find->dirp = readdir(find->hdir)))
if (!STAT(
concat(catbuff, sizeof(catbuff), find->path, find->dirp->d_name),
&find->filestat))
return HTS_TRUE;
if (find->dirp->d_name)
if (!STAT
(concat(catbuff, sizeof(catbuff), find->path, find->dirp->d_name), &find->filestat))
return 1;
#endif
}
return HTS_FALSE;
return 0;
}
HTSEXT_API int hts_findclose(find_handle find) {

View File

@@ -62,7 +62,6 @@ Please visit our Website: http://www.httrack.com
#include "htsmd5.c"
#include "md5.c"
#include "htscmdline.h"
#include "htsserver.h"
#include "htsurlport.h"
#include "htsweb.h"
@@ -89,7 +88,7 @@ Please visit our Website: http://www.httrack.com
static htsmutex refreshMutex = HTSMUTEX_INIT;
static int help_server(char *dest_path, int defaultPort, const char *bindAddr);
static int help_server(char *dest_path, int defaultPort);
extern int commandRunning;
extern int commandEnd;
extern int commandReturn;
@@ -154,8 +153,6 @@ int main(int argc, char *argv[]) {
int ret = 0;
int defaultPort = 0;
int parentPid = 0;
/* loopback by default: the handler trusts its input; --bind widens it */
const char *bindAddr = "127.0.0.1";
printf("Initializing the server..\n");
@@ -182,8 +179,7 @@ int main(int argc, char *argv[]) {
if (argc < 2 || (argc % 2) != 0) {
fprintf(stderr, "** Warning: use the webhttrack frontend if available\n");
fprintf(stderr,
"usage: %s [--port <port>] [--bind <address>] [--ppid parent-pid] "
"<path-to-html-root-dir> [key value [key value]..]\n",
"usage: %s [--port <port>] [--ppid parent-pid] <path-to-html-root-dir> [key value [key value]..]\n",
argv[0]);
fprintf(stderr, "example: %s /usr/share/httrack/\n", argv[0]);
return 1;
@@ -271,14 +267,6 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "couldn't set the port number to %s\n", argv[i + 1]);
return -1;
}
} else if (strcmp(argv[i], "--bind") == 0 && i + 1 < argc) {
/* empty would fall back to every interface, silently undoing the default
*/
if (!strnotempty(argv[i + 1])) {
fprintf(stderr, "--bind needs an address\n");
return -1;
}
bindAddr = argv[i + 1];
} else if (strcmp(argv[i], "--ppid") == 0 && i + 1 < argc) {
if (sscanf(argv[i + 1], "%u", &parentPid) != 1) {
fprintf(stderr, "couldn't set the parent PID to %s\n", argv[i + 1]);
@@ -305,7 +293,7 @@ int main(int argc, char *argv[]) {
}
/* launch */
ret = help_server(argv[1], defaultPort, bindAddr);
ret = help_server(argv[1], defaultPort);
htsthread_wait_n(background_threads - 1);
hts_uninit();
@@ -320,8 +308,10 @@ int main(int argc, char *argv[]) {
static int webhttrack_runmain(httrackp * opt, int argc, char **argv);
static void back_launch_cmd(void *pP) {
char *cmd = (char *) pP;
char **argv;
char **argv = (char **) malloct(1024 * sizeof(char *));
int argc = 0;
int i = 0;
int g = 0;
//
httrackp *opt;
@@ -332,19 +322,28 @@ static void back_launch_cmd(void *pP) {
commandReturnCmdl = strdup(cmd);
/* split */
argv = hts_split_cmdline(cmd, &argc);
if (argv == NULL) {
if (commandReturnMsg)
free(commandReturnMsg);
commandReturnMsg = strdup("could not parse the command line");
commandReturn = -1;
commandRunning = 0;
commandEnd = 1;
free(cmd);
return;
argv[0] = strdup("webhttrack");
argv[1] = cmd;
argc++;
i = 0;
while(cmd[i]) {
if (cmd[i] == '\t' || cmd[i] == '\r' || cmd[i] == '\n') {
cmd[i] = ' ';
}
i++;
}
i = 0;
while(cmd[i]) {
if (cmd[i] == '\"')
g = !g;
if (cmd[i] == ' ') {
if (!g) {
cmd[i] = '\0';
argv[argc++] = cmd + i + 1;
}
}
i++;
}
/* drop the program name the posted command line carries */
argv[0] = strdupt("webhttrack");
/* init */
hts_init();
@@ -373,7 +372,6 @@ static void back_launch_cmd(void *pP) {
/* free */
free(cmd);
freet(argv[0]);
freet(argv);
return;
}
@@ -429,11 +427,11 @@ static int webhttrack_runmain(httrackp * opt, int argc, char **argv) {
return ret;
}
static int help_server(char *dest_path, int defaultPort, const char *bindAddr) {
static int help_server(char *dest_path, int defaultPort) {
int returncode = 0;
char adr_prox[HTS_URLMAXSIZE * 2];
int port_prox;
T_SOC soc = smallserver_init_std(&port_prox, adr_prox, defaultPort, bindAddr);
T_SOC soc = smallserver_init_std(&port_prox, adr_prox, defaultPort);
if (soc != INVALID_SOCKET) {
char url[HTS_URLMAXSIZE * 2];

View File

@@ -46,7 +46,6 @@ Please visit our Website: http://www.httrack.com
#include "httrack.h"
#include "htslib.h"
#include "htscharset.h" // after htslib.h: winsock2.h must precede windows.h
#include "htsbacktrace.h"
/* Static definitions */
static int fexist(const char *s);
@@ -73,6 +72,10 @@ static int linput(FILE * fp, char *s, int max);
#include <sys/ioctl.h>
#endif
#include <ctype.h>
#if (defined(__linux) && defined(HAVE_EXECINFO_H))
#include <execinfo.h>
#define USES_BACKTRACE
#endif
/* END specific definitions */
static void __cdecl htsshow_init(t_hts_callbackarg * carg);
@@ -877,6 +880,21 @@ static void sig_doback(int blind) { // mettre en backing
#undef FD_ERR
#define FD_ERR 2
static void print_backtrace(void) {
#ifdef USES_BACKTRACE
void *stack[256];
const int size = backtrace(stack, sizeof(stack)/sizeof(stack[0]));
if (size != 0) {
backtrace_symbols_fd(stack, size, FD_ERR);
}
#else
const char msg[] = "No stack trace available on this OS :(\n";
if (write(FD_ERR, msg, sizeof(msg) - 1) != sizeof(msg) - 1) {
/* sorry GCC */
}
#endif
}
static size_t print_num(char *buffer, int num) {
size_t i, j;
if (num < 0) {
@@ -910,7 +928,7 @@ static void sig_fatal(int code) {
size += print_num(&buffer[size], code);
buffer[size++] = '\n';
(void) (write(FD_ERR, buffer, size) == size);
hts_print_backtrace(FD_ERR);
print_backtrace();
(void) (write(FD_ERR, msgreport, sizeof(msgreport) - 1)
== sizeof(msgreport) - 1);
abort();
@@ -933,7 +951,6 @@ static void sig_leave(int code) {
}
static void signal_handlers(void) {
hts_backtrace_init();
#ifdef _WIN32
signal(SIGINT, sig_leave); // ^C
signal(SIGTERM, sig_finish); // kill <process>

View File

@@ -100,7 +100,6 @@
<ItemGroup>
<ClCompile Include="httrack.c" />
<ClCompile Include="htsbacktrace.c" />
</ItemGroup>
<!-- Pulls in libhttrack.lib and builds it first. -->

View File

@@ -59,10 +59,9 @@
<ItemDefinitionGroup>
<ClCompile>
<!-- LIBHTTRACK_EXPORTS turns HTSEXT_API into __declspec(dllexport); ZLIB_DLL
imports zlib from its DLL, ZLIB_CONST makes its next_in const as the
autotools build does. HTS_USEBROTLI/HTS_USEZSTD enable the br and zstd
content codings. Windows 7 floor, matching WinHTTrack. -->
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;ZLIB_CONST;HTS_USEBROTLI=1;HTS_USEZSTD=1;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
imports zlib from its DLL. HTS_USEBROTLI/HTS_USEZSTD enable the br and
zstd content codings. Windows 7 floor, matching WinHTTrack. -->
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;HTS_USEBROTLI=1;HTS_USEZSTD=1;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(MSBuildThisFileDirectory);$(MSBuildThisFileDirectory)coucal;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<WarningLevel>Level3</WarningLevel>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
@@ -122,7 +121,6 @@
<ClCompile Include="htshelp.c" />
<ClCompile Include="htsindex.c" />
<ClCompile Include="htslib.c" />
<ClCompile Include="htscmdline.c" />
<ClCompile Include="htsurlport.c" />
<ClCompile Include="htsmd5.c" />
<ClCompile Include="htsmodules.c" />

View File

@@ -771,7 +771,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
PT_ReadIndex(indexes, StringBuff(itemUrl) + 1, FETCH_HEADERS);
if (file != NULL && file->statuscode == HTTP_OK) {
size = file->size;
if (file->lastmodified[0] != '\0') {
if (file->lastmodified) {
timestamp = get_time_rfc822(file->lastmodified);
}
if (timestamp == (time_t) 0) {
@@ -785,7 +785,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
}
timestamp = timestampRep;
}
if (file->contenttype[0] != '\0') {
if (file->contenttype) {
mimeType = file->contenttype;
}
}

View File

@@ -579,10 +579,9 @@ PT_Index PT_LoadCache(const char *filename) {
if (chain != NULL) {
const char *scheme = link_has_authority(chain->name) ? "" : "http://";
/* dropped rather than truncated: empty already reads as "unset" */
if (!sprintfbuff(index->slots.common.startUrl, "%s%s", scheme,
(const char *) chain->name))
index->slots.common.startUrl[0] = '\0';
snprintf(index->slots.common.startUrl,
sizeof(index->slots.common.startUrl), "%s%s", scheme,
(const char *) chain->name);
}
}
}
@@ -866,18 +865,12 @@ static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags)
sprintf(headers + headersSize, "%s: "LLintP"\r\n", field, (LLint)(value)); \
(headersSize) += (int) strlen(headers + headersSize); \
} while(0)
/* refvalue_size is mandatory: value comes off the cache file bounded only by
the line buffer, which dwarfs most destinations. Clipped rather than
rejected, since an engine-written cache carries fields wider than ours
(contenttype is 128 there, 64 here). */
#define ZIP_READFIELD_STRING(line, value, refline, refvalue, refvalue_size) \
do { \
if (line[0] != '\0' && strfield2(line, refline)) { \
(refvalue)[0] = '\0'; \
strlncatbuff(refvalue, value, refvalue_size, (refvalue_size) - 1); \
line[0] = '\0'; \
} \
} while (0)
#define ZIP_READFIELD_STRING(line, value, refline, refvalue) do { \
if (line[0] != '\0' && strfield2(line, refline)) { \
strcpy(refvalue, value); \
line[0] = '\0'; \
} \
} while(0)
#define ZIP_READFIELD_INT(line, value, refline, refvalue) do { \
if (line[0] != '\0' && strfield2(line, refline)) { \
int intval = 0; \
@@ -979,12 +972,9 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
const char *scheme =
link_has_authority(filenameIndex) ? "" : "http://";
/* dropped rather than truncated; try the next entry */
if (sprintfbuff(index->startUrl, "%s%s", scheme,
filenameIndex))
firstSeen = 1;
else
index->startUrl[0] = '\0';
firstSeen = 1;
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
scheme, filenameIndex);
}
}
} else {
@@ -1080,23 +1070,16 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
value++;
ZIP_READFIELD_INT(line, value, "X-In-Cache", dataincache);
ZIP_READFIELD_INT(line, value, "X-Statuscode", r->statuscode);
ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg,
sizeof(r->msg));
ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg); // msg
ZIP_READFIELD_INT(line, value, "X-Size", r->size); // size
ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype,
sizeof(r->contenttype));
ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset,
sizeof(r->charset));
ZIP_READFIELD_STRING(line, value, "Last-Modified",
r->lastmodified, sizeof(r->lastmodified));
ZIP_READFIELD_STRING(line, value, "Etag", r->etag,
sizeof(r->etag));
ZIP_READFIELD_STRING(line, value, "Location", r->location,
sizeof(location_default));
ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype
ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset); // contenttype
ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified
ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag
ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved
ZIP_READFIELD_STRING(line, value, "Content-Disposition",
r->cdispo, sizeof(r->cdispo));
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
sizeof(previous_save_));
r->cdispo); // Content-disposition
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename
if (line[0] != '\0') {
int len = r->headers ? ((int) strlen(r->headers)) : 0;
int nlen =
@@ -1587,11 +1570,9 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
const char *scheme =
link_has_authority(line) ? "" : "http://";
/* dropped rather than truncated; try the next entry */
if (sprintfbuff(index->startUrl, "%s%s", scheme, line))
firstSeen = 1;
else
index->startUrl[0] = '\0';
firstSeen = 1;
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
scheme, line);
}
}

View File

@@ -97,7 +97,6 @@
<ItemGroup>
<ClCompile Include="htsserver.c" />
<ClCompile Include="htsweb.c" />
<ClCompile Include="htscmdline.c" />
<ClCompile Include="htsurlport.c" />
</ItemGroup>

View File

@@ -1,8 +0,0 @@
#!/bin/bash
#
set -euo pipefail
# webhttrack posts its command line as one string: the argv split must grow past
# 1024 arguments and keep a quote inside a value out of the option parser.
httrack -O /dev/null -#test=cmdline-split run | grep -q "cmdline-split self-test OK"

View File

@@ -1,25 +0,0 @@
#!/bin/bash
#
# Buffer capacity for a 64-bit file size ('httrack -#test=growsize'): a -%S list
# file past 4GB must size its buffer exactly or be refused, never wrap.
set -euo pipefail
out=$(httrack -#test=growsize)
echo "$out"
test "$out" == "growsize self-test OK"
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_growsize.XXXXXX")
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
echo '<html><body>hi</body></html>' >"$tmp/index.html"
printf -- '-*/zzmarker*\n' >"$tmp/rules.txt"
# the rules file lands in the URL/filter string, echoed back by the banner
run=$(httrack -O "$tmp/out" --quiet -n "-%S" "$tmp/rules.txt" \
"file://$tmp/index.html" 2>&1) || true
printf '%s\n' "$run" | grep -q 'zzmarker' || {
echo "FAIL: -%S rules file was not loaded"
printf '%s\n' "$run"
exit 1
}

View File

@@ -60,20 +60,13 @@ test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset.
"${python}" - "${url}" "${work}" <<'PY'
import re, sys, urllib.parse, urllib.request
import sys, urllib.parse, urllib.request
url, work = sys.argv[1], sys.argv[2]
outdir = work + "/caf\xe9" # 'café' as the single byte the browser would send
# Port 1 refuses at once: only the decoded -O dir matters, not the fetch.
cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir
# The body is refused without the session id the server renders into each form.
# Note the wizard page is under /server/; a bare /step4.html is the doc page.
page = urllib.request.urlopen(url + "server/step4.html", timeout=20).read()
m = re.search(rb'name="sid" value="([0-9a-f]+)"', page)
if not m:
raise SystemExit("no session id in server/step4.html")
fields = [("sid", m.group(1).decode()),
("path", work), ("projname", "proj"), ("command_do", "start"),
fields = [("path", work), ("projname", "proj"), ("command_do", "start"),
("winprofile", "x"), ("command", cmd)]
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1"))
for k, v in fields)

View File

@@ -1,185 +0,0 @@
#!/bin/bash
#
# htsserver's POST redirect: the Location header is built from a client-supplied
# value, and the listen address is loopback unless --bind widens it.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
# run_with_timeout/kill_tree: timeout(1) is absent on macOS
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
command -v python3 >/dev/null || {
echo "python3 not found; skipping" >&2
exit 77
}
log=$(mktemp)
# start() runs inside a command substitution, so its $! never reaches this
# shell. Take the pid from the server's own announcement instead: a missed kill
# leaves an orphan holding the CI job open long after the suite has passed.
srvpid() { sed -n 's/^PID=//p' "${log}" 2>/dev/null | head -1; }
cleanup() {
stop
rm -f "${log}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
freeport() {
python3 -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()'
}
# Start htsserver, echo the announced URL. Extra args are passed through.
start() {
local port url
port=$(freeport)
: >"${log}"
(
trap '' TERM TTOU
exec htsserver "${distdir}/" --port "${port}" "$@" >"${log}" 2>&1
) &
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
echo "${url}"
}
stop() {
local pid
pid=$(srvpid)
test -z "${pid}" || kill -9 "${pid}" 2>/dev/null || true
}
# The session id gates the request body, so a POST has to carry one. The server
# renders it into every form, which is where a browser picks it up too.
scrape_sid() {
python3 -c 'import socket, sys
s = socket.create_connection(("127.0.0.1", int(sys.argv[1])), 10)
s.settimeout(20)
s.sendall(b"GET /server/index.html HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n")
out = b""
while True:
b = s.recv(65536)
if not b:
break
out += b
s.close()
sys.stdout.write(out.decode("latin-1"))' "$1" |
sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1
}
# POST redirect=$1 to 127.0.0.1:$2 with session id $3, print the raw headers.
post_redirect() {
python3 -c 'import socket, sys, urllib.parse
body = ("sid=" + sys.argv[3] + "&redirect="
+ urllib.parse.quote(sys.argv[1], safe=""))
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
"Content-type: application/x-www-form-urlencoded\r\n"
"Content-length: %d\r\n\r\n%s" % (len(body), body))
s = socket.create_connection(("127.0.0.1", int(sys.argv[2])), 10)
s.settimeout(20)
s.sendall(req.encode())
out = b""
# stop at the end of the header block: the server need not close the connection,
# and an unbounded recv() hung the macOS runner for over an hour.
while b"\r\n\r\n" not in out:
b = s.recv(65536)
if not b:
break
out += b
s.close()
sys.stdout.write(out.split(b"\r\n\r\n")[0].decode("latin-1"))' "$1" "$2" "$3"
}
portof() { echo "${1##*:}" | tr -d /; }
# "free"/"inuse"/"unusable": can $1 still be bound on 127.0.0.2? A wildcard
# listener takes the port on every address, a loopback-only one does not, so
# this discriminates where the announced URL cannot.
probe_alias() {
python3 -c 'import socket, sys
s = socket.socket()
try:
s.bind(("127.0.0.2", int(sys.argv[1])))
except OSError as e:
print("unusable" if e.errno in (99, 49, 10049) else "inuse")
else:
print("free")
finally:
s.close()' "$1"
}
# Non-repeating and not a round number: a truncation, a reorder or a dropped
# interior byte all stay visible.
long=$(python3 -c 'print("".join(chr(33 + i % 90) for i in range(4097)))')
url=$(start)
port=$(portof "${url}")
sid=$(scrape_sid "${port}")
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid (got '${sid}')"
resp=$(post_redirect "${long}" "${port}" "${sid}") ||
fail "no response to an oversized redirect value"
stop
loc=$(printf '%s' "${resp}" | sed -n 's/^Location: //p' | tr -d '\r')
test "${loc}" = "${long}" ||
fail "oversized redirect not echoed whole (got ${#loc} bytes, want ${#long})"
# CR/LF must suppress the header outright. Sanitising it instead would keep the
# grep for an injected header quiet while still emitting a mangled Location.
url=$(start)
port=$(portof "${url}")
sid=$(scrape_sid "${port}")
resp=$(post_redirect '/foo
X-Injected: pwned' "${port}" "${sid}") || fail "no response to a CRLF redirect value"
stop
printf '%s' "${resp}" | grep -qi 'X-Injected' &&
fail "CRLF in the redirect value reached the response"
test "$(printf '%s' "${resp}" | grep -ci '^Location:')" -eq 0 ||
fail "CRLF redirect still emitted a Location header"
test "$(printf '%s' "${resp}" | grep -c '^HTTP/1\.')" -eq 1 ||
fail "CRLF redirect did not yield exactly one status line"
# Loopback unless asked otherwise. Assert the socket, not the announcement.
url=$(start)
port=$(portof "${url}")
alias_state=$(probe_alias "${port}")
stop
if test "${alias_state}" = unusable; then
echo "no 127.0.0.2 alias; skipping the listen-address assertions" >&2
else
test "${alias_state}" = free ||
fail "default listen address is not loopback-only (127.0.0.2:${port} taken)"
case "${url}" in
http://127.0.0.1:*) ;;
*) fail "default announcement is not loopback: ${url}" ;;
esac
url=$(start --bind 0.0.0.0)
port=$(portof "${url}")
alias_state=$(probe_alias "${port}")
stop
test "${alias_state}" = inuse ||
fail "--bind 0.0.0.0 did not take the wildcard (127.0.0.2:${port} ${alias_state})"
fi
# An empty --bind must not quietly fall back to every interface. Bounded: if it
# were accepted the server would listen instead of exiting.
run_with_timeout 15 htsserver "${distdir}/" --bind "" >"${log}" 2>&1 || true
grep -q -- "--bind needs an address" "${log}" ||
fail "empty --bind not refused: $(cat "${log}")"
echo "PASS"

View File

@@ -1,142 +0,0 @@
#!/bin/bash
#
# htsserver's session id must gate the request body: every field in it is
# written into the global key store, and "command" from there reaches the engine.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
command -v python3 >/dev/null || {
echo "python3 not found; skipping" >&2
exit 77
}
srv=
log=$(mktemp)
cleanup() {
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
rm -f "${log}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
freeport() {
python3 -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()'
}
# Echo the announced URL. Runs in a command substitution, so it is a
# subshell and cannot export the pid: the caller reads it back with srvpid.
start() {
local port url
port=$(freeport)
: >"${log}"
(
trap '' TERM TTOU
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
) &
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
echo "${url}"
}
# The server reports its own pid; the aliveness assertions below hang off it.
srvpid() { sed -n 's/^PID=//p' "${log}" | head -1; }
alive() { kill -0 "$1" 2>/dev/null; }
portof() { echo "${1##*:}" | tr -d /; }
# Raw request to 127.0.0.1:$1; $2 is the body ("" for a GET of the $3 page,
# default index). Prints the reply.
request() {
python3 -c 'import socket, sys
port, body = int(sys.argv[1]), sys.argv[2]
page = sys.argv[3] if len(sys.argv) > 3 else "index"
if body:
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
"Content-type: application/x-www-form-urlencoded\r\n"
"Content-length: %d\r\n\r\n%s" % (len(body), body))
else:
req = ("GET /server/%s.html HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % page)
s = socket.create_connection(("127.0.0.1", port), 10)
s.settimeout(30)
s.sendall(req.encode())
out = b""
while True:
b = s.recv(65536)
if not b:
break
out += b
s.close()
sys.stdout.write(out.decode("latin-1"))' "$1" "$2" ${3+"$3"}
}
# The server hands the sid to any client in every form; scraping it is the
# legitimate flow, and it is what makes the accept case below meaningful.
scrape_sid() {
request "$1" "" | sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1
}
url=$(start)
port=$(portof "${url}")
srv=$(srvpid)
test -n "${srv}" || fail "htsserver did not report its pid"
alive "${srv}" || fail "htsserver is not running"
sid=$(scrape_sid "${port}")
# Control: without a real sid every assertion below would pass vacuously.
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid from the page (got '${sid}')"
# Accept: the legitimate flow still works. "redirect" is the cheapest field
# with a reply that is visible in the headers.
resp=$(request "${port}" "sid=${sid}&redirect=/accepted")
printf '%s' "${resp}" | grep -q '^Location: /accepted' ||
fail "a body carrying the correct sid was refused"
# Refuse: missing and wrong. A missing one is the regression under test — the
# expected value used to be pre-seeded into the compared key, so omitting the
# field compared equal to itself.
for bad in "redirect=/nosid" "sid=&redirect=/empty" \
"sid=00000000000000000000000000000000&redirect=/wrong"; do
resp=$(request "${port}" "${bad}")
printf '%s' "${resp}" | grep -q '^Location:' &&
fail "body accepted without a valid sid: ${bad}"
# A refusal has to be a well-formed reply, not a headerless fragment: the
# release build used to emit only Content-length, which reads as a protocol
# error to any client and hides the reason.
printf '%s' "${resp}" | grep -q '^HTTP/1\.0 403 ' ||
fail "refusal was not a 403: ${bad}"
done
# Suppressing the reply is not the same as refusing the write: the body is
# applied to one global key store that later requests render from, and the
# command dispatcher reads it from there. Probe the store itself — step3
# interpolates ${projname} into its title — rather than the refused reply.
title() { request "$1" "" step3; }
request "${port}" "projname=UNAUTHWRITE" >/dev/null 2>&1 || true
title "${port}" | grep -q 'UNAUTHWRITE' &&
fail "a body without a valid sid was written to the key store"
# Paired accept case: without it the assertion above passes even if the server
# simply ignores every body, which would prove nothing.
request "${port}" "sid=${sid}&projname=AUTHWRITE" >/dev/null 2>&1 || true
title "${port}" | grep -q 'AUTHWRITE' ||
fail "a body carrying the correct sid was not written to the key store"
echo "PASS"

View File

@@ -1,90 +0,0 @@
#!/bin/bash
#
# proxytrack's WebDAV PROPFIND fallback must behave identically whether or not
# the cache entry supplies Content-Type and Last-Modified.
set -euo pipefail
: "${top_srcdir:=..}"
# shellcheck source=tests/testlib.sh
. "$top_srcdir/tests/testlib.sh"
python=$(find_python) || {
echo "python3 missing, skipping"
exit 77
}
command -v curl >/dev/null 2>&1 || {
echo "curl missing, skipping"
exit 77
}
# First test to run proxytrack as a live server, and MSYS cannot reap a native
# listener: the orphan wedged the whole Windows suite for 48 minutes (#595).
if is_windows; then
echo "windows: cannot reap a backgrounded proxytrack, skipping"
exit 77
fi
dir=$(mktemp -d)
ptpid=
cleanup() {
stop_server "$ptpid"
rm -rf "$dir"
}
trap cleanup EXIT
# Neither header is present, so file->contenttype and ->lastmodified stay "".
printf 'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
printf 'hello' >"$dir/body"
alen=$(($(wc -c <"$dir/hdr") + $(wc -c <"$dir/body")))
{
printf 'filedesc://t.arc 0.0.0.0 20250101000000 text/plain 200 - - 0 t.arc 9\n'
printf '2 0 test\n'
printf '\n\n'
printf 'http://example.com/page.html 0.0.0.0 20250101000000 text/html 200 - - 0 t.arc %d\n' "$alen"
cat "$dir/hdr" "$dir/body"
} >"$dir/in.arc"
freeport() {
"$python" -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()'
}
proxyport=$(freeport)
icpport=$(freeport)
proxytrack "127.0.0.1:$proxyport" "127.0.0.1:$icpport" "$dir/in.arc" >"$dir/pt.log" 2>&1 &
ptpid=$!
waited=0
until grep -qE "HTTP Proxy installed on|Unable to (initialize a temporary server|create the server)" "$dir/pt.log"; do
kill -0 "$ptpid" 2>/dev/null || {
echo "FAIL: proxytrack exited before listening"
cat "$dir/pt.log"
exit 1
}
test "$waited" -lt 50 || {
echo "FAIL: proxytrack never announced its listen port"
exit 1
}
sleep 0.1
waited=$((waited + 1))
done
grep -q "HTTP Proxy installed on" "$dir/pt.log" || {
echo "FAIL: proxytrack failed to bind"
cat "$dir/pt.log"
exit 1
}
# --max-time: an unbounded read here would wedge the runner rather than fail
curl -s --max-time 30 -X PROPFIND -H "Depth: 1" \
"http://127.0.0.1:$proxyport/webdav/example.com/" >"$dir/resp.xml"
grep -q '<getcontenttype>application/octet-stream</getcontenttype>' "$dir/resp.xml" || {
echo "FAIL: missing Content-Type did not fall back to application/octet-stream"
cat "$dir/resp.xml"
exit 1
}
grep -q '<getlastmodified>Wed, 01 Jan 2025 00:00:00 GMT</getlastmodified>' "$dir/resp.xml" || {
echo "FAIL: missing Last-Modified did not fall back to the index timestamp"
cat "$dir/resp.xml"
exit 1
}
echo "OK: WebDAV listing falls back correctly for an entry with no Content-Type/Last-Modified"

View File

@@ -1,97 +0,0 @@
#!/bin/bash
# A fatal signal prints the raw backtrace, then names the frames
# backtrace_symbols_fd() leaves as module+offset (-fvisibility=hidden hides them).
set -eu
testdir=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
if is_windows; then
echo "no backtrace() on Windows; skipping" >&2
exit 77
fi
command -v httrack >/dev/null || {
echo "could not find httrack" >&2
exit 1
}
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_symbolize.XXXXXX") || exit 1
trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
out="${tmpdir}/trace"
raw="${tmpdir}/trace-optout"
overflow="this string is far too long for the buffer"
# A frame glibc could not name. Stop at the offset: it prints the trailing
# "[0xADDR]" with or without a leading space depending on its version.
rawframe='(+0x[0-9a-f][0-9a-f]*)'
# The strsafe selftest aborts on purpose, which lands in sig_fatal.
rc=0
run_with_timeout 60 httrack -#test=strsafe overflow "$overflow" >"$out" 2>&1 || rc=$?
test "$rc" -ne 124 || {
echo "the crash handler did not finish within the deadline" >&2
exit 1
}
grep -q '^Caught signal ' "$out" || {
echo "no 'Caught signal' line:" >&2
cat "$out" >&2
exit 1
}
if grep -q 'No stack trace available on this OS' "$out"; then
echo "no backtrace() on this platform; skipping" >&2
exit 77
fi
# Unconditional and first: symbolizing must never cost the raw trace.
grep -q "$rawframe" "$out" || {
echo "raw module+offset frames are gone:" >&2
cat "$out" >&2
exit 1
}
command -v addr2line >/dev/null || {
echo "addr2line not installed; skipping" >&2
exit 77
}
# Control: resolve the same frames ourselves, so a stripped build cannot turn
# the assertion below into a vacuous pass.
oracle="${tmpdir}/oracle"
sed -n 's/^\([^(]*\)(+\(0x[0-9a-f]*\)).*$/\1 \2/p' "$out" |
while read -r mod off; do
test -f "$mod" || continue
addr2line -Cfip -e "$mod" "$off" 2>/dev/null || true
done >"$oracle"
grep -qE 'st_strsafe|strcpy_safe_' "$oracle" || {
echo "addr2line names no hidden symbol in this build; skipping" >&2
exit 77
}
# The payload. Dropping the raw frames first is what makes it specific: both
# names are static, so .dynsym could never have carried them. Not anchored on
# the "0xOFF:" line, the inline chain puts the name on either half.
grep -v "$rawframe" "$out" | grep -qE 'st_strsafe|strcpy_safe_' || {
echo "the handler did not name the hidden frames:" >&2
cat "$out" >&2
exit 1
}
# Opt-out: raw trace kept, nothing spawned.
export HTTRACK_NO_SYMBOLIZE=1
rc=0
run_with_timeout 60 httrack -#test=strsafe overflow "$overflow" >"$raw" 2>&1 || rc=$?
unset HTTRACK_NO_SYMBOLIZE
test "$rc" -ne 124 || {
echo "the opt-out run did not finish within the deadline" >&2
exit 1
}
grep -q "$rawframe" "$raw" || {
echo "the opt-out lost the raw trace:" >&2
cat "$raw" >&2
exit 1
}
! grep -q '^0x[0-9a-f]*: ' "$raw" || {
echo "the opt-out still symbolized:" >&2
cat "$raw" >&2
exit 1
}

View File

@@ -1,142 +0,0 @@
#!/bin/bash
#
# The wizard's size fields must render as distinct caps: sizemax is -M, othermax
# then maxhtml share -m.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
python=$(find_python) || {
echo "python3 not found; skipping" >&2
exit 77
}
work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_maxsize.XXXXXX") || fail "no tmpdir"
srvlog=$(mktemp)
srv=
cleanup() {
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
srv=
rm -rf "${work}" "${srvlog}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
# ~/.httrack.ini out of it.
sport=$("${python}" -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()')
(
trap '' TERM TTOU
export HOME="${work}"
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
) &
srv=$!
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
kill -0 "${srv}" 2>/dev/null || break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
# Audit what step4.html renders back; without command_do nothing is launched.
"${python}" - "${url}" <<'PY' || fail "wizard rendered the wrong options (see above)"
import re, sys, urllib.parse, urllib.request
url = sys.argv[1]
html, other, site = "111111", "222222", "333333"
def post(fields):
# The body is refused without the session id the server puts in each form.
form = urllib.request.urlopen(url + "server/index.html", timeout=20).read()
m = re.search(rb'name="sid" value="([0-9a-f]+)"', form)
if not m:
raise SystemExit("no session id in server/index.html")
fields = [("sid", m.group(1).decode())] + fields
body = "&".join("%s=%s" % (k, urllib.parse.quote(v)) for k, v in fields)
req = urllib.request.Request(url + "server/step4.html",
data=body.encode("latin-1"), method="POST")
return urllib.request.urlopen(req, timeout=20).read().decode("latin-1")
def textarea(page, name):
m = re.search(r'<textarea name="%s".*?>(.*?)</textarea>' % name, page, re.S)
if m is None:
sys.exit("no %s textarea in the rendered step4.html" % name)
return m.group(1)
def want(ok, msg, ctx):
if not ok:
sys.exit("%s\nrendered:%s" % (msg, ctx))
page = post([("maxhtml", html), ("othermax", other), ("sizemax", site),
("dos", "2")])
cmd = textarea(page, "command")
want("--max-size=" + site in cmd, "site cap not emitted as --max-size", cmd)
want("--max-files=" + site not in cmd, "site cap still an --max-files", cmd)
# Positive controls: both per-file caps still ride -m, one option each.
want("--max-files=" + other in cmd, "non-HTML cap not emitted as --max-files",
cmd)
want("--max-files=," + html in cmd, "HTML cap not emitted as --max-files=,",
cmd)
want(cmd.count("--max-files=") == 2, "unexpected --max-files count", cmd)
want(cmd.index("--max-files=" + other) < cmd.index("--max-files=," + html),
"the bare --max-files must precede the --max-files=, form", cmd)
# winprofile.ini feeds the same three caps to the Windows GUI, one key each.
ini = textarea(page, "winprofile").replace("\r\n", "\n") # the ini is CRLF
want("\nDos=2" in ini, "Dos not written from the dos field", ini)
want("\nMaxHtml=" + html + "\n" in ini, "MaxHtml not written from maxhtml", ini)
want("\nMaxOther=" + other + "\n" in ini, "MaxOther not written from othermax",
ini)
want("\nMaxAll=" + site + "\n" in ini, "MaxAll not written from sizemax", ini)
# A bare --max-files= (an unguarded empty field) parses as -m with no digits,
# silently clearing the html cap.
page = post([("maxhtml", ""), ("othermax", ""), ("sizemax", ""), ("dos", "2")])
cmd = textarea(page, "command")
want("--max-rate=" in cmd, "empty size fields rendered no command line", cmd)
want("--max-files" not in cmd and "--max-size" not in cmd,
"empty size fields still rendered a size option", cmd)
# A mistyped ${LANG_OK] key renders the OK button's label empty.
opts = urllib.request.urlopen(url + "server/option2b.html",
timeout=20).read().decode("latin-1")
want("LANG_OK" not in opts, "unexpanded LANG_OK in option2b.html", "")
want('<input type="submit" value="OK"' in opts, "no OK label in option2b.html",
"")
PY
cleanup
# A bare -m<n> resets the html limit, so only the bare-then-comma order keeps
# both caps live. basic.html is 487 bytes, well past the 10-byte html cap.
crawl() {
bash "${distdir}/tests/local-crawl.sh" "$@"
}
crawl --errors 1 --log-found 'File too big' \
httrack 'BASEURL/simple/basic.html' '--max-files=,10'
crawl --errors 1 --log-found 'File too big' \
httrack 'BASEURL/simple/basic.html' '--max-files=500000' '--max-files=,10'
crawl --errors 0 --found 'simple/basic.html' --log-not-found 'File too big' \
httrack 'BASEURL/simple/basic.html' '--max-files=,10' '--max-files=500000'
echo "PASS"

View File

@@ -1,134 +0,0 @@
#!/bin/bash
#
# A browser will not follow an http: page to a file: URL, so the GUI's "browse
# the mirror" link has to go through the server's own /website/ route.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
python=$(find_python) || {
echo "python3 not found; skipping" >&2
exit 77
}
# Control: on a path that does not resolve, grep "passes" without reading anything.
test -f "${distdir}/html/server/finished.html" || fail "no GUI pages under ${distdir}"
bad=$(grep -l 'file://' "${distdir}"/html/server/finished.html || true)
test -z "${bad}" || fail "file: link left in: ${bad}"
work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_browse.XXXXXX") || fail "no tmpdir"
srvlog=$(mktemp)
srv=
srvpid=
cleanup() {
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
test -z "${srvpid}" || kill -9 "${srvpid}" 2>/dev/null || true
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
rm -rf "${work}" "${srvlog}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
# Stand in for a finished mirror: the crawl itself is not under test.
proj="${work}/websites/proj"
mkdir -p "${proj}"
printf 'MIRROR-PROBE-OK\n' >"${proj}/probe.txt"
printf '<html><body>MIRROR-INDEX-OK</body></html>\n' >"${proj}/index.html"
# An isolated HOME keeps a stray ~/.httrack.ini out of the server's settings.
sport=$("${python}" -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()')
(
trap '' TERM TTOU
export HOME="${work}"
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
) &
srv=$!
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
kill -0 "${srv}" 2>/dev/null || break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
srvpid=$(sed -n 's/^PID=//p' "${srvlog}") # absent on Windows
# htsserver resolves the posted paths itself, so hand it native ones.
"${python}" - "${url}" "$(nativepath "${work}/websites")" "$(nativepath "${proj}")" <<'PY' || fail "browse-link checks failed"
import re, sys, urllib.error, urllib.parse, urllib.request
url, base, proj = sys.argv[1].rstrip("/"), sys.argv[2], sys.argv[3]
rc = 0
def check(ok, what):
global rc
print(("ok: " if ok else "FAIL: ") + what)
if not ok:
rc = 1
def get(path):
# The UI is served ISO-8859-1, so stay in bytes.
return urllib.request.urlopen(url + path, timeout=20).read()
# No crawl has run, so no commandRunning/commandEnd override swaps the page out.
finished = get("/server/finished.html").decode("latin-1")
# Control: an empty or unrendered page would pass "no file:" on its own.
check("HTTrack Website Copier" in finished, "finished.html rendered")
check("file://" not in finished, "finished.html has no file: link")
# Control: /website/ 404s until a project is set, so the fetches below are real.
try:
get("/website/probe.txt")
check(False, "/website/ served with no project set")
except urllib.error.HTTPError as e:
check(e.code == 404, "/website/ is bound to a project (got %d)" % e.code)
# Point the server at the mirror the way step4.html does; a body without the
# session id is refused outright.
sid = re.search(r'name="sid" value="([0-9a-f]+)"', finished)
if sid is None:
print("FAIL: no sid in the rendered form")
sys.exit(1)
fields = [("sid", sid.group(1)), ("path", base), ("projname", "proj"),
("projpath", proj + "/")]
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="")) for k, v in fields)
urllib.request.urlopen(urllib.request.Request(
url + "/server/step4.html", data=body.encode("latin-1"), method="POST"),
timeout=20).read()
check(get("/website/probe.txt") == b"MIRROR-PROBE-OK\n", "mirror file over http")
check("MIRROR-INDEX-OK" in get("/website/index.html").decode("latin-1"),
"mirror index over http")
body = get("/server/finished.html").decode("latin-1")
# Match the anchor by its mirror-path label: the list below it links /website/
# too, so a bare "is the route mentioned" check passes on the unfixed page.
check(re.search(r'href="/website/index\.html"[^>]*>\s*' +
re.escape(base + "/proj"), body) is not None,
"the mirror-path link points at the served mirror")
check("file://" not in body, "finished.html has no file: link")
sys.exit(rc)
PY
# A leaked htsserver wedges the parallel harness behind a green log.
cleanup
! kill -0 "${srv}" 2>/dev/null || fail "htsserver ${srv} survived"
echo "PASS"

View File

@@ -1,148 +0,0 @@
#!/bin/bash
#
# The wizard renders its httrack command line as one string, which the engine
# splits back into argv: a double quote in a field must reach the split escaped,
# or the rest of the value is parsed as fresh options (-V runs a shell command).
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
command -v python3 >/dev/null || {
echo "python3 not found; skipping" >&2
exit 77
}
srv=
log=$(mktemp)
cleanup() {
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
rm -f "${log}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
freeport() {
python3 -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()'
}
# Echo the announced URL. Runs in a command substitution, so it is a
# subshell and cannot export the pid: the caller reads it back with srvpid.
start() {
local port url
port=$(freeport)
: >"${log}"
(
trap '' TERM TTOU
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
) &
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
echo "${url}"
}
srvpid() { sed -n 's/^PID=//p' "${log}" | head -1; }
portof() { echo "${1##*:}" | tr -d /; }
# Raw request to 127.0.0.1:$1; $2 is the body ("" for a GET of the $3 page,
# default index). Prints the reply.
request() {
python3 -c 'import socket, sys
port, body = int(sys.argv[1]), sys.argv[2]
page = sys.argv[3] if len(sys.argv) > 3 else "index"
if body:
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
"Content-type: application/x-www-form-urlencoded\r\n"
"Content-length: %d\r\n\r\n%s" % (len(body), body))
else:
req = ("GET /server/%s.html HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % page)
s = socket.create_connection(("127.0.0.1", port), 10)
s.settimeout(30)
s.sendall(req.encode())
out = b""
while True:
b = s.recv(65536)
if not b:
break
out += b
s.close()
sys.stdout.write(out.decode("latin-1"))' "$1" "$2" ${3+"$3"}
}
scrape_sid() {
request "$1" "" | sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1
}
# urlencode the key=value pairs given as arguments
formencode() {
python3 -c 'import sys, urllib.parse
print(urllib.parse.urlencode([tuple(a.split("=", 1)) for a in sys.argv[1:]]))' "$@"
}
url=$(start)
port=$(portof "${url}")
srv=$(srvpid)
test -n "${srv}" || fail "htsserver did not report its pid"
sid=$(scrape_sid "${port}")
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid from the page (got '${sid}')"
# Fill the wizard fields the command line quotes, then read back the generated
# command line: the user-agent carries a break-out attempt, the footer a
# backslash (which must survive the escape round trip), the project name a plain
# value.
body=$(formencode "sid=${sid}" 'user=Moz" -V "touch /tmp/pwn' 'footer=a\b"c' \
"path=/tmp/p" "projname=plain proj" 'urls=http://x/a"b' 'url2=+*.png"')
request "${port}" "${body}" >/dev/null
cmdline=$(request "${port}" "" step4 |
sed -n '/<textarea name="command"/,/<\/textarea>/p')
# Control: without the fields in the page every assertion below is vacuous.
grep -q -- '--user-agent' <<<"${cmdline}" ||
fail "no --user-agent in the generated command line (probe blind)"
# A plain value is passed through untouched: the escaping must not mangle the
# ordinary case.
grep -qF -- '--path "/tmp/p/plain proj"' <<<"${cmdline}" ||
fail "a plain quoted value was not passed through: ${cmdline}"
# The quote is escaped, so the split keeps it inside the value...
grep -qF -- '--user-agent "Moz\" -V \"touch /tmp/pwn"' <<<"${cmdline}" ||
fail "the quote in the user-agent was not escaped: ${cmdline}"
# ...and the raw form, which would end the argument and hand -V to the option
# parser, is gone.
grep -qF -- '--user-agent "Moz" -V "touch' <<<"${cmdline}" &&
fail "the user-agent still closes its argument early: ${cmdline}"
# A backslash is escaped too, or the split would eat it along with the quote
# that follows.
grep -qF -- '--footer "a\\b\"c"' <<<"${cmdline}" ||
fail "the backslash in the footer was not escaped: ${cmdline}"
# The url and filter fields sit outside quotes, where a backslash cannot escape
# anything: one quote there flips the parity of every quote after it, so the
# escaping above would protect nothing. They must not emit a raw quote at all.
grep -qF -- 'http://x/a%22b' <<<"${cmdline}" ||
fail "the quote in the url field was not neutralised: ${cmdline}"
grep -qF -- '+*.png%22' <<<"${cmdline}" ||
fail "the quote in the filter field was not neutralised: ${cmdline}"
grep -qF -- 'http://x/a"b' <<<"${cmdline}" &&
fail "the url field still emits a raw quote: ${cmdline}"
echo "PASS"

View File

@@ -1,167 +0,0 @@
#!/bin/bash
#
# htsserver serves the crawled mirror under /website/ alongside its own GUI:
# mirrored pages must skip the ${...} expander, or ${_sid} leaks the session id.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
command -v python3 >/dev/null || {
echo "python3 not found; skipping" >&2
exit 77
}
log=$(mktemp)
work=$(mktemp -d)
csrv=
# start() runs in a command substitution, so its $! never reaches this shell. A
# missed kill leaves an orphan holding the CI job open long after a green suite.
srvpid() { sed -n 's/^PID=//p' "${log}" 2>/dev/null | head -1; }
cleanup() {
local pid
pid=$(srvpid)
test -z "${pid}" || kill -9 "${pid}" 2>/dev/null || true
test -z "${csrv}" || kill -9 "${csrv}" 2>/dev/null || true
wait "${csrv}" 2>/dev/null || true # absorb bash's async "Killed" notice
rm -rf "${log}" "${work}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
freeport() {
python3 -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()'
}
# Echo the announced URL.
start() {
local port url
port=$(freeport)
: >"${log}"
(
trap '' TERM TTOU
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
) &
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
echo "${url}"
}
portof() { echo "${1##*:}" | tr -d /; }
# GET $2 from 127.0.0.1:$1, headers into $3 and the body, byte for byte, into $4.
fetch() {
python3 -c 'import socket, sys
s = socket.create_connection(("127.0.0.1", int(sys.argv[1])), 10)
s.settimeout(20)
s.sendall(("GET %s HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % sys.argv[2]).encode())
out = b""
while True:
b = s.recv(65536)
if not b:
break
out += b
s.close()
head, _, body = out.partition(b"\r\n\r\n")
open(sys.argv[3], "wb").write(head)
open(sys.argv[4], "wb").write(body)' "$1" "$2" "$3" "$4"
}
# POST the remaining args as urlencoded key=value fields to 127.0.0.1:$1.
post() {
local port=$1
shift
python3 -c 'import socket, sys, urllib.parse
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe=""))
for k, v in (a.split("=", 1) for a in sys.argv[2:])).encode()
s = socket.create_connection(("127.0.0.1", int(sys.argv[1])), 10)
s.settimeout(30)
s.sendall(b"POST /step4.html HTTP/1.0\r\nHost: 127.0.0.1\r\n"
b"Content-type: application/x-www-form-urlencoded\r\n"
b"Content-length: %d\r\n\r\n" % len(body) + body)
while s.recv(65536):
pass
s.close()' "${port}" "$@" >/dev/null
}
# LF-only, and a line ending in a backslash: the expander rewrites both, so a
# mangled reply fails the byte comparison even where no directive is present.
mirror="${work}/proj/hostile.html"
hdr="${work}/hdr"
body="${work}/body"
# The server merges $HOME/.httrack.ini into the same store on the first request;
# point it somewhere empty so a developer's own file cannot shadow the fields.
export HOME="${work}"
url=$(start)
port=$(portof "${url}")
# Positive control: the GUI's own templates must still expand. This is also
# where the real token comes from, so its absence below can be asserted.
fetch "${port}" /server/index.html "${hdr}" "${body}"
sid=$(sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' "${body}" | head -1)
test "${#sid}" -eq 32 || fail "GUI page did not expand \${sid} (got '${sid}')"
# /website/ serves only the root the server itself recorded, so save a profile
# (no command_do=start, so nothing crawls) to create it, then plant the file.
post "${port}" "sid=${sid}" command=httrack command_do=save winprofile=x \
"path=${work}" projname=proj
test -f "${work}/proj/hts-cache/winprofile.ini" ||
fail "profile save did not create the project: $(cat "${log}")"
mkdir -p "$(dirname "${mirror}")"
# shellcheck disable=SC2016 # the directives are the payload, not shell expansions
printf 'Hostile mirrored page.\nsid=${_sid} copy=${sid}\ntrailing backslash: \\\n' \
>"${mirror}"
fetch "${port}" /website/hostile.html "${hdr}" "${body}"
grep -q '^HTTP/1\.0 200 ' "${hdr}" || fail "mirrored page not served: $(head -1 "${hdr}")"
grep -qF "${sid}" "${body}" &&
fail "the session id was expanded into mirrored content"
# shellcheck disable=SC2016 # the directives are the payload, not shell expansions
grep -qF '${_sid}' "${body}" || fail "\${_sid} did not survive verbatim"
# shellcheck disable=SC2016
grep -qF '${sid}' "${body}" || fail "\${sid} did not survive verbatim"
cmp -s "${mirror}" "${body}" || fail "mirrored file not served byte for byte"
# The mirror stays browsable: verbatim must not mean served as a download.
grep -qi '^Content-type: text/html' "${hdr}" ||
fail "mirrored page lost its text/html type: $(cat "${hdr}")"
# The other direction: while a crawl runs, every .html request is overridden to
# the GUI's own refresh page, so a /website/ URL stops naming mirrored content.
# /trickle/ dribbles for a minute, which holds the crawl open for the probe.
clog="${work}/content.log"
python3 "${testdir}/local-server.py" --root "${work}" >"${clog}" 2>&1 &
csrv=$!
for _ in $(seq 1 40); do
cport=$(sed -n 's/^PORT //p' "${clog}") && test -n "${cport}" && break
kill -0 "${csrv}" 2>/dev/null || break
sleep 0.25
done
test -n "${cport:-}" || fail "content server did not come up: $(cat "${clog}")"
post "${port}" "sid=${sid}" "path=${work}" projname=crawl winprofile=x \
command_do=start \
"command=httrack --quiet --robots=0 http://127.0.0.1:${cport}/trickle/ -O ${work}/crawl"
fetch "${port}" /website/hostile.html "${hdr}" "${body}"
grep -q '^HTTP/1\.0 200 ' "${hdr}" ||
fail "running crawl: /website/ was not overridden to the GUI page: $(head -1 "${hdr}")"
grep -qF "'crawl' - HTTrack Website Copier" "${body}" ||
fail "the overridden GUI page was not the expanded refresh page"
echo "PASS"

View File

@@ -1,168 +0,0 @@
#!/bin/bash
#
# /website/ is served from the project directory htsserver set up itself, never
# from a root the request body names, and composing that path must stay bounded.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
command -v python3 >/dev/null || {
echo "python3 not found; skipping" >&2
exit 77
}
srv=
log=$(mktemp)
base=$(mktemp -d)
cleanup() {
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
rm -f "${log}"
rm -rf "${base}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
freeport() {
python3 -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()'
}
# Echo the announced URL. Runs in a command substitution, so it is a
# subshell and cannot export the pid: the caller reads it back with srvpid.
start() {
local port url
port=$(freeport)
: >"${log}"
(
trap '' TERM TTOU
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
) &
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
echo "${url}"
}
# The server reports its own pid; the aliveness assertion below hangs off it.
srvpid() { sed -n 's/^PID=//p' "${log}" | head -1; }
alive() { kill -0 "$1" 2>/dev/null; }
portof() { echo "${1##*:}" | tr -d /; }
# Raw request to 127.0.0.1:$1: GET the path $2, or POST the body $3 to / when
# $2 is empty. Prints the reply.
request() {
python3 -c 'import socket, sys
port, path, body = int(sys.argv[1]), sys.argv[2], sys.argv[3]
if path:
req = "GET %s HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n" % path
else:
req = ("POST / HTTP/1.0\r\nHost: 127.0.0.1\r\n"
"Content-type: application/x-www-form-urlencoded\r\n"
"Content-length: %d\r\n\r\n%s" % (len(body), body))
s = socket.create_connection(("127.0.0.1", port), 10)
s.settimeout(30)
s.sendall(req.encode())
out = b""
while True:
b = s.recv(65536)
if not b:
break
out += b
s.close()
sys.stdout.write(out.decode("latin-1"))' "$1" "$2" "$3"
}
get() { request "$1" "$2" ""; }
post() { request "$1" "" "$2"; }
url=$(start)
port=$(portof "${url}")
srv=$(srvpid)
test -n "${srv}" || fail "htsserver did not report its pid"
# Every request body is gated by the session id (78_webhttrack-sid.test).
sid=$(get "${port}" /server/index.html |
sed -n 's/.*name="sid" value="\([0-9a-f]*\)".*/\1/p' | head -1)
test "${#sid}" -eq 32 || fail "did not scrape a 32-hex sid (got '${sid}')"
# A file the mirror must never expose, next to the project that may.
echo "SECRETMARKER" >"${base}/secret.txt"
mkdir -p "${base}/proj"
echo "LOGMARKER" >"${base}/proj/hts-log.txt"
# error_redirect is the branch that skipped the fsfile clearing, so fail the save
# with a component over NAME_MAX (mkdir refuses it whatever the uid). Must precede
# any successful save: commandEnd then swaps the error page for the finished one.
get "${port}" /server/style.css >/dev/null # leaves a path behind in fsfile
post "${port}" "sid=${sid}&command=httrack&command_do=save&winprofile=x&path=${base}/$(printf '%0300d' 0)&projname=p" |
grep -q '^Location: /server/error.html' ||
fail "the refused save did not redirect to the error page"
# No project yet, so no root to serve from.
post "${port}" "sid=${sid}&projpath=${base}/" >/dev/null
get "${port}" /website/secret.txt | grep -q SECRETMARKER &&
fail "a posted projpath served a file outside any project"
# Positive control: step4's "save settings" flow registers the project without
# crawling, and browsing its mirror keeps working. Without it the assertions
# around it would pass on a server that never serves /website/ at all.
body="sid=${sid}&command=httrack&command_do=save&winprofile=x"
body="${body}&path=${base}&projname=proj&projpath=${base}/proj/"
post "${port}" "${body}" >/dev/null
test -f "${base}/proj/hts-cache/winprofile.ini" ||
fail "the project was not registered: $(cat "${log}")"
get "${port}" /website/hts-log.txt | grep -q LOGMARKER ||
fail "the registered project's mirror is not served"
# Same request with the root repointed: the project is legitimate, projpath is
# not what decides where the bytes come from.
post "${port}" "sid=${sid}&projpath=${base}/" >/dev/null
get "${port}" /website/secret.txt | grep -q SECRETMARKER &&
fail "a posted projpath repointed the served root"
post "${port}" "sid=${sid}&projpath=/etc/" >/dev/null
get "${port}" /website/passwd | grep -q '^root:' &&
fail "a posted projpath read an arbitrary system file"
# A ".." anywhere in the recorded root would escape the mirror on every later
# request, so the save must be refused and the previous root kept.
post "${port}" "sid=${sid}&command=httrack&command_do=save&winprofile=x&path=${base}/proj&projname=.." >/dev/null
get "${port}" /website/secret.txt | grep -q SECRETMARKER &&
fail "a '..' in the saved project path escaped the mirror root"
get "${port}" /website/hts-log.txt | grep -q LOGMARKER ||
fail "rejecting the '..' root also lost the previous one"
# The root is now the server's own, but it is still built from two posted
# fields: 800-odd bytes of them used to be sprintf'd into a 1024-byte buffer.
seg=$(printf '%0200d' 0)
longpath="${base}/${seg}/${seg}/${seg}/${seg}"
fspath="${longpath}/proj"
# structcheck() refuses a root over HTS_URLMAXSIZE, so the URL carries the rest.
test "$((${#fspath} + 11))" -le 1024 ||
fail "the long project path (${#fspath}) would not pass structcheck"
longurl=$(printf '%0800d' 0)
test "$((${#fspath} + 1 + ${#longurl}))" -gt 1024 ||
fail "the composed path (${#fspath} + ${#longurl}) would not overflow"
post "${port}" "sid=${sid}&command=httrack&command_do=save&winprofile=x&path=${longpath}&projname=proj" >/dev/null
test -f "${fspath}/hts-cache/winprofile.ini" ||
fail "the long-path project was not registered: $(cat "${log}")"
get "${port}" "/website/${longurl}" >/dev/null 2>&1 || true
alive "${srv}" || fail "an over-long project path crashed the server: $(cat "${log}")"
get "${port}" /server/index.html | grep -q '200 OK' ||
fail "the server stopped answering after the over-long project path"
echo "PASS"

View File

@@ -1,61 +0,0 @@
#!/bin/bash
# A cached header longer than proxytrack's field must be clipped, not copied
# raw: contenttype[64] sits just before the location pointer, so an unbounded
# copy smashed it and the next Location line wrote through the wreckage.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
python=$(find_python) || {
echo "python3 not found; skipping" >&2
exit 77
}
dir=$(mktemp -d)
trap 'rm -rf "$dir"' EXIT
# proxytrack reads its metadata from each entry's ZIP local extra field
"$python" - "$dir/in.zip" <<'EOF'
import sys, zipfile
body = b"hello world"
ct = "text/html" + "A" * 400
meta = (
"X-In-Cache: 1\r\n"
"X-StatusCode: 200\r\n"
"X-StatusMessage: OK\r\n"
"X-Size: %d\r\n"
"Content-Type: %s\r\n"
"X-Charset: iso-8859-1\r\n"
# a parseable date is required: the ARC writer dereferences it unchecked
"Last-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\n"
"Location: http://example.com/%s\r\n"
"X-Save: out/page.html\r\n" % (len(body), ct, "B" * 400)
).encode()
zi = zipfile.ZipInfo("example.com/page.html")
zi.compress_type = zipfile.ZIP_STORED
zi.extra = meta
with zipfile.ZipFile(sys.argv[1], "w") as z:
z.writestr(zi, body)
EOF
proxytrack --convert "$dir/out.arc" "$dir/in.zip" >/dev/null 2>&1 || {
echo "FAIL: proxytrack crashed on a cache with over-long header fields" >&2
exit 1
}
test -s "$dir/out.arc" || {
echo "FAIL: entry dropped instead of clipped" >&2
exit 1
}
# clipped to contenttype[64], so 63 characters survive: "text/html" plus 54 A's
longest=$(grep -ao 'A\+' "$dir/out.arc" | awk '{ print length($0) }' | sort -rn | head -n1)
test "$longest" = 54 || {
echo "FAIL: expected the content-type clipped to 63 chars (54 A's), got ${longest:-none}" >&2
exit 1
}

View File

@@ -32,7 +32,6 @@ TESTS = \
00_runnable.test \
01_engine-charset.test \
01_engine-cmdline.test \
01_engine-cmdline-split.test \
01_engine-cookies.test \
01_engine-copyopt.test \
01_engine-crange.test \
@@ -65,7 +64,6 @@ TESTS = \
01_engine-reconcile.test \
01_engine-expandhome.test \
01_engine-fsize.test \
01_engine-growsize.test \
01_engine-redirect.test \
01_engine-longpath-io.test \
01_engine-mirror-io.test \
@@ -173,16 +171,6 @@ TESTS = \
74_local-warc-wacz.test \
74_local-warc-verbatim.test \
75_engine-longpath-posix.test \
76_cli-resize.test \
77_webhttrack-redirect.test \
78_webhttrack-sid.test \
79_local-proxytrack-webdav-mime.test \
80_engine-crash-symbolize.test \
81_webhttrack-maxsize.test \
82_webhttrack-browse-links.test \
83_webhttrack-argescape.test \
84_webhttrack-mirror-verbatim.test \
85_webhttrack-projpath.test \
86_local-proxytrack-cache-longfields.test
76_cli-resize.test
CLEANFILES = check-network_sh.cache