mirror of
https://github.com/xroche/httrack.git
synced 2026-07-16 13:51:46 +03:00
Compare commits
8 Commits
feature/en
...
proxy-conn
| Author | SHA1 | Date | |
|---|---|---|---|
|
c40bc14cec | ||
|
|
29e868b2ca | ||
|
|
f1418bac3c | ||
|
|
092a557305 | ||
|
|
cfe7c7e68f | ||
|
|
89b1e485d7 | ||
|
|
08481464bd | ||
|
|
3e7b9f91ee |
6
.github/workflows/codeql.yml
vendored
6
.github/workflows/codeql.yml
vendored
@@ -46,6 +46,12 @@ jobs:
|
||||
# fopen's umask-controlled 0666 is intended for mirror/cache/log
|
||||
# output; the one credential file (cookies.txt) is kept 0600 on Unix.
|
||||
config: |
|
||||
paths-ignore:
|
||||
# Demo callback samples, not part of libhttrack.
|
||||
- libtest
|
||||
# ProxyTrack: a separate legacy binary with no auth surface; its
|
||||
# recv/cache-parse code trips taint queries by design.
|
||||
- src/proxy
|
||||
query-filters:
|
||||
- exclude:
|
||||
id: cpp/world-writable-file-creation
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
|
||||
if FUZZERS
|
||||
noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
|
||||
fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
|
||||
fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx \
|
||||
fuzz-htsparse
|
||||
endif
|
||||
|
||||
AM_CPPFLAGS = \
|
||||
@@ -25,6 +26,7 @@ fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
|
||||
fuzz_url_SOURCES = fuzz-url.c fuzz.h
|
||||
fuzz_header_SOURCES = fuzz-header.c fuzz.h
|
||||
fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
|
||||
fuzz_htsparse_SOURCES = fuzz-htsparse.c fuzz.h
|
||||
|
||||
# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
|
||||
EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
@@ -37,9 +39,12 @@ EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
corpus/filters/filter.bin corpus/filters/filter-size.bin \
|
||||
corpus/filters/regress-empty-subject-unique.bin \
|
||||
corpus/filters/redos-star-classes.bin \
|
||||
corpus/filters/regress-classdepth-timeout.bin \
|
||||
corpus/url/http-url.txt corpus/url/relative-path.txt \
|
||||
corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
|
||||
corpus/header/full-response.txt corpus/header/redirect.txt \
|
||||
corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
|
||||
corpus/cachendx/regress-overadvance.bin \
|
||||
corpus/cachendx/regress-truncated-entry.bin
|
||||
corpus/cachendx/regress-truncated-entry.bin \
|
||||
corpus/htsparse/basic.html corpus/htsparse/script-inscript.html \
|
||||
corpus/htsparse/meta-usemap.html corpus/htsparse/malformed.html
|
||||
|
||||
BIN
fuzz/corpus/filters/regress-classdepth-timeout.bin
Normal file
BIN
fuzz/corpus/filters/regress-classdepth-timeout.bin
Normal file
Binary file not shown.
15
fuzz/corpus/htsparse/basic.html
Normal file
15
fuzz/corpus/htsparse/basic.html
Normal file
@@ -0,0 +1,15 @@
|
||||
<!DOCTYPE html>
|
||||
<html><head>
|
||||
<meta charset="utf-8">
|
||||
<base href="http://example.com/dir/">
|
||||
<title>Seed</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<script src="app.js"></script>
|
||||
</head><body>
|
||||
<h1>Hi</h1>
|
||||
<a href="page2.html">next</a>
|
||||
<a href="http://other.example.org/x?y=1#frag">abs</a>
|
||||
<img src="pic.png" srcset="a.png 1x, b.png 2x">
|
||||
<script>var u="inline.html"; document.write('<a href="gen.html">g</a>');</script>
|
||||
<form action="/cgi/submit"><input name="q"></form>
|
||||
</body></html>
|
||||
8
fuzz/corpus/htsparse/malformed.html
Normal file
8
fuzz/corpus/htsparse/malformed.html
Normal file
@@ -0,0 +1,8 @@
|
||||
<a href="unclosed.html
|
||||
<img src='mix"ed.png>
|
||||
<a href=noquote.html >bare</a>
|
||||
<!-- comment <a href="incomment.html"> -->
|
||||
<a href="tab newline
|
||||
.html">wsp</a>
|
||||
<script>unterminated "string and <a href=
|
||||
<a href=
|
||||
11
fuzz/corpus/htsparse/meta-usemap.html
Normal file
11
fuzz/corpus/htsparse/meta-usemap.html
Normal file
@@ -0,0 +1,11 @@
|
||||
<html><head>
|
||||
<meta http-equiv="refresh" content="0; url=redir.html">
|
||||
<base href="http://h/b/">
|
||||
<link rel="alternate" href="feed.xml">
|
||||
</head><body background="page-bg.jpg">
|
||||
<img src="map.png" usemap="#m">
|
||||
<map name="m"><area href="area1.html" coords="0,0,10,10"></map>
|
||||
<applet code="A.class" codebase="applets/"><param name="src" value="p.dat"></applet>
|
||||
<object data="o.swf"><embed src="e.svg"></object>
|
||||
<a href="http://ent/e.html">entity</a>
|
||||
</body></html>
|
||||
13
fuzz/corpus/htsparse/script-inscript.html
Normal file
13
fuzz/corpus/htsparse/script-inscript.html
Normal file
@@ -0,0 +1,13 @@
|
||||
<html><body>
|
||||
<script type="text/javascript">
|
||||
var a = "http://x/1.html", b = 'q\'uote', c = "/*not*/comment";
|
||||
// line "with" quotes and http://y/2.html
|
||||
/* block 'with' <a href="notparsed"> */
|
||||
var re = /a\/b\//g;
|
||||
document.write('<img src="w1.png">');
|
||||
document.writeln("<a href='w2.html'>k</a>");
|
||||
element.onclick = "location='onh.html'";
|
||||
</script>
|
||||
<a href="after.html" onmouseover="go('ev.html')">x</a>
|
||||
<div style="background:url(bg.png)">z</div>
|
||||
</body></html>
|
||||
175
fuzz/fuzz-htsparse.c
Normal file
175
fuzz/fuzz-htsparse.c
Normal file
@@ -0,0 +1,175 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the real htsparse() over a mocked engine: the minimal crawl state
|
||||
httpmirror() builds, then a page walked through the parser and discarded. The
|
||||
str/stre wiring below mirrors htsparse()'s call site in htscore.c; update it
|
||||
in lockstep if those structs gain a field the parser reads. */
|
||||
#include "fuzz.h"
|
||||
|
||||
#include "httrack-library.h"
|
||||
#include "htscore.h"
|
||||
#include "htsback.h"
|
||||
#include "htshash.h"
|
||||
#include "htsrobots.h"
|
||||
#include "htsparse.h"
|
||||
#include "htsmodules.h"
|
||||
#include "coucal.h"
|
||||
|
||||
/* htsparse ignores str.addLink on the internal parse; stub it. */
|
||||
static int fuzz_addlink(htsmoduleStruct *str, char *link) {
|
||||
(void) str;
|
||||
(void) link;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static int inited = 0;
|
||||
|
||||
httrackp *opt;
|
||||
cache_back cache;
|
||||
hash_struct hash;
|
||||
robots_wizard robots;
|
||||
struct_back *sback;
|
||||
char **filters = NULL;
|
||||
int filptr = 0;
|
||||
|
||||
htsblk r;
|
||||
htsmoduleStruct str;
|
||||
htsmoduleStructExtended stre;
|
||||
int ptr, error = 0, store_errpage = 0;
|
||||
int makeindex_done = 0, makeindex_links = 0;
|
||||
FILE *makeindex_fp = NULL;
|
||||
char makeindex_firstlink[HTS_URLMAXSIZE * 2] = "";
|
||||
LLint stat_fragment = 0, makestat_total = 0;
|
||||
int makestat_lnk = 0;
|
||||
char base[HTS_URLMAXSIZE * 2] = "";
|
||||
char codebase[HTS_URLMAXSIZE * 2] = "";
|
||||
char err_msg[1024] = "";
|
||||
|
||||
if (!inited) {
|
||||
hts_init();
|
||||
inited = 1;
|
||||
}
|
||||
|
||||
opt = hts_create_opt();
|
||||
opt->log = opt->errlog = NULL;
|
||||
opt->robots = 0;
|
||||
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 0; /* no on-disk cache */
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.cached_tests = coucal_new(0);
|
||||
coucal_value_is_malloc(cache.cached_tests, 1);
|
||||
|
||||
memset(&robots, 0, sizeof(robots));
|
||||
strcpybuff(robots.adr, "!");
|
||||
opt->robotsptr = &robots;
|
||||
|
||||
opt->maxfilter = maximum(opt->maxfilter, 128);
|
||||
filters_init(&filters, opt->maxfilter, 0);
|
||||
opt->filters.filters = &filters;
|
||||
opt->filters.filptr = &filptr;
|
||||
|
||||
opt->hash = &hash;
|
||||
hts_record_init(opt);
|
||||
hash_init(opt, &hash, opt->urlhack);
|
||||
hash.liens = (const lien_url *const *const *) &opt->liens;
|
||||
|
||||
sback = back_new(opt, opt->maxsoc * 32 + 1024);
|
||||
|
||||
/* ptr=1 (index 1 is the parsed page) selects full HTML parsing + the
|
||||
rewriter; urladr()/urlfil()/savename() alias heap(ptr), save=/dev/null. */
|
||||
hts_record_link(opt, "example.com", "/", "/dev/null", "", "", NULL);
|
||||
hts_record_link(opt, "example.com", "/index.html", "/dev/null", "", "", NULL);
|
||||
ptr = 1;
|
||||
|
||||
/* NUL-terminated in a size+1 alloc: htsparse one-past-reads onto the NUL. */
|
||||
hts_init_htsblk(&r);
|
||||
r.statuscode = 200;
|
||||
r.size = (LLint) size;
|
||||
r.adr = malloct(size + 1);
|
||||
if (size)
|
||||
memcpy(r.adr, data, size);
|
||||
r.adr[size] = '\0';
|
||||
strcpybuff(r.contenttype, "text/html");
|
||||
|
||||
memset(&str, 0, sizeof(str));
|
||||
memset(&stre, 0, sizeof(stre));
|
||||
str.err_msg = err_msg;
|
||||
str.filename = heap(ptr)->sav;
|
||||
str.mime = r.contenttype;
|
||||
str.url_host = heap(ptr)->adr;
|
||||
str.url_file = heap(ptr)->fil;
|
||||
str.size = (int) r.size;
|
||||
str.addLink = fuzz_addlink;
|
||||
str.opt = opt;
|
||||
str.sback = sback;
|
||||
str.cache = &cache;
|
||||
str.hashptr = &hash;
|
||||
str.numero_passe = 0;
|
||||
str.ptr_ = &ptr;
|
||||
str.page_charset_ = NULL;
|
||||
|
||||
stre.r_ = &r;
|
||||
stre.error_ = &error;
|
||||
stre.exit_xh_ = &opt->state.exit_xh;
|
||||
stre.store_errpage_ = &store_errpage;
|
||||
stre.base = base;
|
||||
stre.codebase = codebase;
|
||||
stre.filters_ = &filters;
|
||||
stre.filptr_ = &filptr;
|
||||
stre.robots_ = &robots;
|
||||
stre.hash_ = &hash;
|
||||
stre.makeindex_done_ = &makeindex_done;
|
||||
stre.makeindex_fp_ = &makeindex_fp;
|
||||
stre.makeindex_links_ = &makeindex_links;
|
||||
stre.makeindex_firstlink_ = makeindex_firstlink;
|
||||
stre.template_header_ = "";
|
||||
stre.template_body_ = "";
|
||||
stre.template_footer_ = "";
|
||||
stre.stat_fragment_ = &stat_fragment;
|
||||
stre.makestat_time = 0;
|
||||
stre.makestat_fp = NULL;
|
||||
stre.makestat_total_ = &makestat_total;
|
||||
stre.makestat_lnk_ = &makestat_lnk;
|
||||
stre.maketrack_fp = NULL;
|
||||
|
||||
(void) htsparse(&str, &stre);
|
||||
|
||||
freet(r.adr);
|
||||
back_delete_all(opt, &cache, sback);
|
||||
back_free(&sback);
|
||||
hash_free(&hash);
|
||||
coucal_delete(&cache.hashtable);
|
||||
coucal_delete(&cache.cached_tests);
|
||||
checkrobots_free(&robots);
|
||||
if (filters != NULL) {
|
||||
if (filters[0] != NULL)
|
||||
freet(filters[0]);
|
||||
freet(filters);
|
||||
}
|
||||
hts_record_free(opt);
|
||||
hts_free_opt(opt);
|
||||
return 0;
|
||||
}
|
||||
@@ -98,7 +98,16 @@ ${do:end-if}
|
||||
<input type="hidden" name="redirect" value="">
|
||||
<input type="hidden" name="closeme" value="">
|
||||
|
||||
${LANG_IOPT10}:
|
||||
${LANG_PROXYTYPE}
|
||||
<select name="proxytype"
|
||||
onMouseOver="info('${html:LANG_PROXYTYPETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
>
|
||||
<option value=""${ztest:proxytype: selected:}>HTTP</option>
|
||||
<option value="1"${test:proxytype:: selected}>SOCKS5</option>
|
||||
</select>
|
||||
<br><br>
|
||||
|
||||
${LANG_IOPT10}:
|
||||
<input name="prox" value="${prox}" size="32"
|
||||
onMouseOver="info('${html:LANG_G14}'); return true" onMouseOut="info(' '); return true"
|
||||
>:
|
||||
|
||||
@@ -148,6 +148,14 @@ ${LANG_I65}
|
||||
>
|
||||
</td></tr>
|
||||
|
||||
<tr><td>
|
||||
${LANG_PAUSEFILES}
|
||||
</td><td>
|
||||
<input name="pausefiles" value="${pausefiles}" size="8"
|
||||
onMouseOver="info('${html:LANG_PAUSEFILESTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
>
|
||||
</td></tr>
|
||||
|
||||
<tr><td>
|
||||
${LANG_I52}
|
||||
</td><td>
|
||||
|
||||
@@ -103,6 +103,12 @@ ${do:end-if}
|
||||
> ${LANG_I58}
|
||||
<br><br>
|
||||
|
||||
${LANG_COOKIEFILE}
|
||||
<input name="cookiesfile" value="${cookiesfile}" size="40"
|
||||
onMouseOver="info('${html:LANG_COOKIEFILETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
>
|
||||
<br><br>
|
||||
|
||||
${LANG_I59}
|
||||
<br>
|
||||
<select name="checktype"
|
||||
@@ -136,6 +142,27 @@ ${listid:robots:LISTDEF_8}
|
||||
> ${LANG_I62b2}
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="keepwww" ${checked:keepwww}
|
||||
onMouseOver="info('${html:LANG_KEEPWWWTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_KEEPWWW}
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="keepslashes" ${checked:keepslashes}
|
||||
onMouseOver="info('${html:LANG_KEEPSLASHESTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_KEEPSLASHES}
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="keepqueryorder" ${checked:keepqueryorder}
|
||||
onMouseOver="info('${html:LANG_KEEPQUERYORDERTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_KEEPQUERYORDER}
|
||||
<br><br>
|
||||
|
||||
${LANG_STRIPQUERY}
|
||||
<input name="stripquery" value="${stripquery}" size="40"
|
||||
onMouseOver="info('${html:LANG_STRIPQUERYTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
>
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="toler" ${checked:toler}
|
||||
onMouseOver="info('${html:LANG_I1i}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I62}
|
||||
|
||||
@@ -129,15 +129,21 @@ ${do:copy:NoPwdInPages:hidepwd}
|
||||
${do:copy:NoQueryStrings:hidequery}
|
||||
${do:copy:NoPurgeOldFiles:nopurge}
|
||||
${do:copy:Cookies:cookies}
|
||||
${do:copy:CookiesFile:cookiesfile}
|
||||
${do:copy:CheckType:checktype}
|
||||
${do:copy:ParseJava:parsejava}
|
||||
${do:copy:HTTP10:http10}
|
||||
${do:copy:TolerantRequests:toler}
|
||||
${do:copy:UpdateHack:updhack}
|
||||
${do:copy:URLHack:urlhack}
|
||||
${do:copy:KeepWww:keepwww}
|
||||
${do:copy:KeepSlashes:keepslashes}
|
||||
${do:copy:KeepQueryOrder:keepqueryorder}
|
||||
${do:copy:StripQuery:stripquery}
|
||||
${do:copy:StoreAllInCache:cache2}
|
||||
${do:copy:LogType:logtype}
|
||||
${do:copy:UseHTTPProxyForFTP:ftpprox}
|
||||
${do:copy:ProxyType:proxytype}
|
||||
${do:copy:Build:build}
|
||||
${do:copy:PrimaryScan:filter}
|
||||
${do:copy:Travel:travel}
|
||||
@@ -148,6 +154,7 @@ ${do:copy:MaxHtml:maxhtml}
|
||||
${do:copy:MaxOther:othermax}
|
||||
${do:copy:MaxAll:sizemax}
|
||||
${do:copy:MaxWait:pausebytes}
|
||||
${do:copy:PauseFiles:pausefiles}
|
||||
${do:copy:Sockets:connexion}
|
||||
${do:copy:Retry:retry}
|
||||
${do:copy:MaxTime:maxtime}
|
||||
|
||||
@@ -178,6 +178,12 @@ ${do:end-if}
|
||||
${test:parsejava:--parse-java=0:}
|
||||
${test:updhack:--updatehack}
|
||||
${test:urlhack:--urlhack=0:--urlhack}
|
||||
${test:keepwww:--keep-www-prefix}
|
||||
${test:keepslashes:--keep-double-slashes}
|
||||
${test:keepqueryorder:--keep-query-order}
|
||||
${test:cookiesfile:--cookies-file "}${html:cookiesfile}${test:cookiesfile:"}
|
||||
${test:pausefiles:--pause "}${pausefiles}${test:pausefiles:"}
|
||||
${test:stripquery:--strip-query "}${html:stripquery}${test:stripquery:"}
|
||||
${test:toler:--tolerant}
|
||||
${test:http10:--http-10}
|
||||
${test:cache2:--store-all-in-cache}
|
||||
@@ -186,7 +192,7 @@ ${do:end-if}
|
||||
${test:logtype:::--extra-log:--debug-log}
|
||||
${test:index:--index=0:}
|
||||
${test:index2:--search-index=0:--search-index}
|
||||
${test:prox:--proxy "}${prox}${test:prox:\3A}${portprox}${test:prox:"}
|
||||
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype:socks5\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
|
||||
${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
|
||||
</textarea>
|
||||
|
||||
@@ -219,15 +225,21 @@ NoPwdInPages=${ztest:hidepwd:0:1}
|
||||
NoQueryStrings=${ztest:hidequery:0:1}
|
||||
NoPurgeOldFiles=${ztest:nopurge:0:1}
|
||||
Cookies=${ztest:cookies:0:1}
|
||||
CookiesFile=${cookiesfile}
|
||||
CheckType=${ztest:checktype:0:1:2}
|
||||
ParseJava=${ztest:parsejava:0:1}
|
||||
HTTP10=${ztest:http10:0:1}
|
||||
TolerantRequests=${ztest:toler:0:1}
|
||||
UpdateHack=${ztest:updhack:0:1}
|
||||
URLHack=${ztest:urlhack:0:1}
|
||||
KeepWww=${ztest:keepwww:0:1}
|
||||
KeepSlashes=${ztest:keepslashes:0:1}
|
||||
KeepQueryOrder=${ztest:keepqueryorder:0:1}
|
||||
StripQuery=${stripquery}
|
||||
StoreAllInCache=${ztest:cache2:0:1}
|
||||
LogType=${logtype}
|
||||
UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
|
||||
ProxyType=${proxytype}
|
||||
Build=${build}
|
||||
PrimaryScan=${filter}
|
||||
Travel=${travel}
|
||||
@@ -238,6 +250,7 @@ MaxHtml=${maxhtml}
|
||||
MaxOther=${othermax}
|
||||
MaxAll=${sizemax}
|
||||
MaxWait=${pausebytes}
|
||||
PauseFiles=${pausefiles}
|
||||
Sockets=${connexion}
|
||||
Retry=${retry}
|
||||
MaxTime=${maxtime}
|
||||
|
||||
Submodule src/coucal updated: fadf29bd2a...ab59c6a7a0
@@ -2696,9 +2696,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
int dispo = 0;
|
||||
|
||||
// probe the resolved address list once per fresh connect (cache hit:
|
||||
// the host was resolved when this connect was opened)
|
||||
// the host was resolved when this connect was opened). Not under a
|
||||
// proxy: the socket dials the proxy, so resolving the origin here leaks
|
||||
// its DNS and lets a proxy-connect failure fall back to dialing it
|
||||
// direct.
|
||||
if (cf->addr_count < 0 && back[i].r.soc != INVALID_SOCKET &&
|
||||
!back[i].r.is_file) {
|
||||
!back[i].r.is_file && !back[i].r.req.proxy.active) {
|
||||
SOCaddr scratch[HTS_MAXADDRNUM];
|
||||
|
||||
cf->addr_count = hts_dns_resolve_all(opt, back[i].url_adr, scratch,
|
||||
@@ -3310,16 +3313,26 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].chunk_blocksize = -1; /* ending */
|
||||
back[i].r.totalsize += chunk_size; // noter taille
|
||||
if (back[i].r.adr != NULL || !back[i].r.is_write) { // Not to disk
|
||||
back[i].r.adr =
|
||||
(char *) realloct(back[i].r.adr,
|
||||
(size_t) back[i].r.totalsize + 1);
|
||||
if (!back[i].r.adr) {
|
||||
if (cache->log != NULL) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"not enough memory (" LLintP
|
||||
") for %s%s",
|
||||
(LLint) back[i].r.totalsize,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
// totalsize sums attacker-declared chunk sizes; past
|
||||
// 2GB the (size_t) cast below truncates on 32-bit and
|
||||
// under-allocates. Mark the chunk invalid so the shared
|
||||
// error path tears the transfer down.
|
||||
if (back[i].r.totalsize > INT32_MAX) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Chunked resource too large for %s%s",
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
chunk_size = -1;
|
||||
} else {
|
||||
back[i].r.adr = (char *) realloct(
|
||||
back[i].r.adr, (size_t) back[i].r.totalsize + 1);
|
||||
if (!back[i].r.adr) {
|
||||
if (cache->log != NULL) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"not enough memory (" LLintP
|
||||
") for %s%s",
|
||||
(LLint) back[i].r.totalsize,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,12 +154,14 @@ static const char *strjoker_rec(strjoker_memo *memo, const char *chaine,
|
||||
|
||||
if (depth > memo->maxdepth)
|
||||
memo->maxdepth = depth;
|
||||
/* Charge the budget before the depth cap: cut disables the memo, so uncounted
|
||||
deep calls would let the sub-cap search explode (OSS-Fuzz 535114376). */
|
||||
if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
|
||||
return NULL; /* work budget spent: fail the match safely */
|
||||
if (depth >= STRJOKER_MAXDEPTH) {
|
||||
memo->cut = HTS_TRUE;
|
||||
return NULL; /* nesting beyond any real filter: fail the branch safely */
|
||||
}
|
||||
if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
|
||||
return NULL; /* work budget spent: fail the match safely */
|
||||
if (memo->failed) {
|
||||
bit = (size_t) (chaine - memo->chaine0) * memo->stride +
|
||||
(size_t) (joker - memo->joker0);
|
||||
|
||||
@@ -6030,6 +6030,8 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
|
||||
StringFree(opt->referer);
|
||||
StringFree(opt->from);
|
||||
StringFree(opt->lang_iso);
|
||||
StringFree(opt->accept);
|
||||
StringFree(opt->headers);
|
||||
StringFree(opt->sys_com);
|
||||
StringFree(opt->mimedefs);
|
||||
StringFree(opt->filelist);
|
||||
|
||||
@@ -794,6 +794,20 @@ static int st_filterbounds(httrackp *opt, int argc, char **argv) {
|
||||
memset(subj, 'a', 32);
|
||||
subj[32] = '\0';
|
||||
assertf(strjoker(subj, pat, NULL, NULL) != NULL);
|
||||
/* Same pin for the class-branch shape users actually write (*[..]), against a
|
||||
long subject: it must match with room to spare under the work cap. */
|
||||
{
|
||||
const char *seg = "*[A-Z,a-z,0-9]";
|
||||
const size_t seglen = strlen(seg), nseg = 16;
|
||||
|
||||
for (i = 0; i < (int) nseg; i++)
|
||||
memcpy(pat + i * seglen, seg, seglen);
|
||||
pat[nseg * seglen] = '\0';
|
||||
memset(subj, 'a', 512);
|
||||
subj[512] = '\0';
|
||||
assertf(strjoker_bounds(subj, pat, &steps, &maxsteps, NULL, NULL) != NULL);
|
||||
assertf(steps < maxsteps);
|
||||
}
|
||||
freet(pat);
|
||||
freet(subj);
|
||||
printf("filterbounds: OK\n");
|
||||
|
||||
18
tests/55_local-chunked.test
Executable file
18
tests/55_local-chunked.test
Executable file
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
# A well-formed Transfer-Encoding: chunked response mirrors intact: the chunk
|
||||
# automaton (htsback.c) joins the bodies and the 2GB in-RAM cap stays quiet.
|
||||
# The 76-char run spans a 64-byte chunk boundary (only contiguous once joined);
|
||||
# a CR anywhere would be leftover framing; '</body>' proves no truncation.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'chunked/page.html' \
|
||||
--file-matches 'chunked/page.html' \
|
||||
'chunk-body chunk-body chunk-body chunk-body chunk-body chunk-body chunk-body' \
|
||||
--file-matches 'chunked/page.html' '</body>' \
|
||||
--file-not-matches 'chunked/page.html' $'\r' \
|
||||
--log-not-found 'too large' \
|
||||
httrack 'BASEURL/chunked/index.html'
|
||||
93
tests/56_local-proxy-noleak.test
Normal file
93
tests/56_local-proxy-noleak.test
Normal file
@@ -0,0 +1,93 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A crawl through a proxy must never resolve or dial the origin itself: the proxy
|
||||
# does that. A dead proxy + a multi-address origin used to fall back to dialing
|
||||
# the origin direct (bypassing the proxy, leaking its DNS and IP). The decoy
|
||||
# origin here must therefore receive nothing.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
if test "${V6_SUPPORT:-}" == "no"; then
|
||||
echo "no IPv6 support (resolver override compiled out), skipping"
|
||||
exit 77
|
||||
fi
|
||||
python=$(find_python) || {
|
||||
echo "python3 missing, skipping"
|
||||
exit 77
|
||||
}
|
||||
|
||||
server=$(nativepath "$top_srcdir/tests/local-server.py")
|
||||
root=$(nativepath "$top_srcdir/tests/server-root")
|
||||
tmpdir=$(mktemp -d)
|
||||
serverpid=
|
||||
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
return 0
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# decoy origin: it must stay silent. LOCAL_SERVER_VERBOSE logs any request it gets.
|
||||
LOCAL_SERVER_VERBOSE=1 "$python" "$server" --root "$root" --bind 127.0.0.1 \
|
||||
>"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$tmpdir/srv.out" 2>/dev/null || true)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$tmpdir/srv.err")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# a proxy port nothing listens on: grab a free one and let it close (refuses).
|
||||
deadproxy=$("$python" -c \
|
||||
'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()')
|
||||
|
||||
# origin resolves to two addresses so a fallback would have a second to dial;
|
||||
# 127.0.0.1 (the decoy) is the one the old bypass reached.
|
||||
out="$tmpdir/crawl"
|
||||
HTTRACK_DEBUG_RESOLVE="decoyhost:127.0.0.2,127.0.0.1" \
|
||||
httrack "http://decoyhost:$port/simple/basic.html" -O "$out" \
|
||||
-P "127.0.0.1:$deadproxy" -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z \
|
||||
>"$tmpdir/log" 2>&1
|
||||
|
||||
log="$out/hts-log.txt"
|
||||
|
||||
# the origin must have seen no connection (no leak/bypass)
|
||||
if grep -qE '"(GET|POST|HEAD|CONNECT) ' "$tmpdir/srv.err"; then
|
||||
echo "FAIL: origin was contacted directly, bypassing the proxy"
|
||||
cat "$tmpdir/srv.err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# the crawl must fail at the proxy (proves it really tried the proxy, not a no-op)
|
||||
grep -q 'Connect Error' "$log" || {
|
||||
echo "FAIL: expected a proxy connect error"
|
||||
cat "$log"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# and it must not have tried an origin-address fallback under the proxy
|
||||
if grep -q "trying next address" "$log"; then
|
||||
echo "FAIL: fell back to an origin address under a proxy"
|
||||
cat "$log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: a dead proxy fails cleanly without dialing the origin"
|
||||
@@ -131,6 +131,8 @@ TESTS = \
|
||||
51_local-update-codec.test \
|
||||
52_local-socks5.test \
|
||||
53_local-proxytrack-cache-corrupt.test \
|
||||
54_local-update-truncate-purge.test
|
||||
54_local-update-truncate-purge.test \
|
||||
55_local-chunked.test \
|
||||
56_local-proxy-noleak.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -1215,6 +1215,28 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def route_chunked_index(self):
|
||||
self.send_html('\t<a href="page.html">chunked</a>\n')
|
||||
|
||||
def route_chunked_page(self):
|
||||
# Transfer-Encoding: chunked over many small chunks: drives the engine's
|
||||
# chunk automaton (htsback.c). The mirrored file must equal the joined
|
||||
# chunk bodies, so the 2GB in-RAM cap doesn't fire on ordinary traffic.
|
||||
blob = big_html("chunked", "<p>" + "chunk-body " * 300 + "</p>")
|
||||
self.protocol_version = "HTTP/1.1"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Transfer-Encoding", "chunked")
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if self.command == "HEAD":
|
||||
return
|
||||
step = 64
|
||||
for off in range(0, len(blob), step):
|
||||
piece = blob[off : off + step]
|
||||
self.wfile.write(b"%X\r\n" % len(piece) + piece + b"\r\n")
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
|
||||
# Content-Disposition naming: the attachment filename replaces the
|
||||
# URL-derived name; path components in it are stripped (RFC 2616).
|
||||
CDISPO_NAMES = {
|
||||
@@ -1554,6 +1576,8 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/crange206mem/blob.bin": route_crange206mem,
|
||||
"/size/index.html": route_size_index,
|
||||
"/size/oversize.bin": route_size_oversize,
|
||||
"/chunked/index.html": route_chunked_index,
|
||||
"/chunked/page.html": route_chunked_page,
|
||||
"/errpage/index.html": route_errpage_index,
|
||||
"/errpage/good.html": route_errpage_good,
|
||||
"/errpage/missing.html": route_errpage_missing,
|
||||
|
||||
Reference in New Issue
Block a user