Compare commits

...

3 Commits

Author SHA1 Message Date
Xavier Roche
d69b0b1426 CodeQL: exclude libtest demos and src/proxy from scanning
libtest/ holds demo callback samples that aren't part of libhttrack, and
src/proxy/ is ProxyTrack, a separate legacy binary with no auth surface whose
recv/cache-parse code trips the taint queries by design. Together they account
for ~26 alerts that are false-positive or by-design. Exclude both so the
dashboard tracks the engine, where the queries earn their keep.

Genuine bugs found in that code are fixed on their own, independent of scanning.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-15 21:08:41 +02:00
Xavier Roche
092a557305 Bound strjoker's work budget across the depth cap (fuzz-filters timeout, OSS-Fuzz 535114376) (#590)
strjoker_rec charged its work budget only after the depth-cap early-return, so depth-capped recursive calls cost nothing. A class pattern like *[a]*[a]... drives recursion to STRJOKER_MAXDEPTH, sets the sticky cut flag (disabling the failure memo), and the band below the cap then re-explores exponentially without ever tripping the budget — a ~2KB filter ran ~80s under ASan. Moving the increment ahead of the depth-cap return makes every call count, bounding total work; the match fails safe. Behavior is unchanged for real filters. Adds a fuzzer corpus regression seed and a class-branch lower-bound pin to the filterbounds self-test.
2026-07-15 20:32:56 +02:00
Xavier Roche
cfe7c7e68f Expose post-3.49-2 engine options in webhttrack (#589)
The web GUI was frozen before the engine gained SOCKS5 proxying, --cookies-file,
--pause, --strip-query, and the three --keep-* URL-hack opt-outs. Wire each into
the option tabs, the generated httrack command, the winprofile.ini persistence,
and the reload remap so they reach parity with WinHTTrack (xroche/httrack-windows#31).

The SOCKS5 selector reuses the single proxy field and prepends socks5:// to -P;
it is gated on a non-empty host so a SOCKS5-selected-but-blank config can't emit a
stray scheme token. winprofile.ini key names match WinHTTrack so projects round-trip
across both GUIs. Labels/tooltips reuse the LANG_ strings already added in #588.

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-15 20:17:03 +02:00
10 changed files with 91 additions and 4 deletions

View File

@@ -46,6 +46,12 @@ jobs:
# fopen's umask-controlled 0666 is intended for mirror/cache/log
# output; the one credential file (cookies.txt) is kept 0600 on Unix.
config: |
paths-ignore:
# Demo callback samples, not part of libhttrack.
- libtest
# ProxyTrack: a separate legacy binary with no auth surface; its
# recv/cache-parse code trips taint queries by design.
- src/proxy
query-filters:
- exclude:
id: cpp/world-writable-file-creation

View File

@@ -39,6 +39,7 @@ EXTRA_DIST = README.md run-fuzzers.sh \
corpus/filters/filter.bin corpus/filters/filter-size.bin \
corpus/filters/regress-empty-subject-unique.bin \
corpus/filters/redos-star-classes.bin \
corpus/filters/regress-classdepth-timeout.bin \
corpus/url/http-url.txt corpus/url/relative-path.txt \
corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
corpus/header/full-response.txt corpus/header/redirect.txt \

Binary file not shown.

View File

@@ -98,7 +98,16 @@ ${do:end-if}
<input type="hidden" name="redirect" value="">
<input type="hidden" name="closeme" value="">
${LANG_IOPT10}:
${LANG_PROXYTYPE}
<select name="proxytype"
onMouseOver="info('${html:LANG_PROXYTYPETIP}'); return true" onMouseOut="info('&nbsp;'); return true"
>
<option value=""${ztest:proxytype: selected:}>HTTP</option>
<option value="1"${test:proxytype:: selected}>SOCKS5</option>
</select>
<br><br>
${LANG_IOPT10}:
<input name="prox" value="${prox}" size="32"
onMouseOver="info('${html:LANG_G14}'); return true" onMouseOut="info('&nbsp;'); return true"
>:

View File

@@ -148,6 +148,14 @@ ${LANG_I65}
>
</td></tr>
<tr><td>
${LANG_PAUSEFILES}
</td><td>
<input name="pausefiles" value="${pausefiles}" size="8"
onMouseOver="info('${html:LANG_PAUSEFILESTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
>
</td></tr>
<tr><td>
${LANG_I52}
</td><td>

View File

@@ -103,6 +103,12 @@ ${do:end-if}
> ${LANG_I58}
<br><br>
${LANG_COOKIEFILE}
<input name="cookiesfile" value="${cookiesfile}" size="40"
onMouseOver="info('${html:LANG_COOKIEFILETIP}'); return true" onMouseOut="info('&nbsp;'); return true"
>
<br><br>
${LANG_I59}
<br>
<select name="checktype"
@@ -136,6 +142,27 @@ ${listid:robots:LISTDEF_8}
> ${LANG_I62b2}
<br><br>
<input type="checkbox" name="keepwww" ${checked:keepwww}
onMouseOver="info('${html:LANG_KEEPWWWTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
> ${LANG_KEEPWWW}
<br><br>
<input type="checkbox" name="keepslashes" ${checked:keepslashes}
onMouseOver="info('${html:LANG_KEEPSLASHESTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
> ${LANG_KEEPSLASHES}
<br><br>
<input type="checkbox" name="keepqueryorder" ${checked:keepqueryorder}
onMouseOver="info('${html:LANG_KEEPQUERYORDERTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
> ${LANG_KEEPQUERYORDER}
<br><br>
${LANG_STRIPQUERY}
<input name="stripquery" value="${stripquery}" size="40"
onMouseOver="info('${html:LANG_STRIPQUERYTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
>
<br><br>
<input type="checkbox" name="toler" ${checked:toler}
onMouseOver="info('${html:LANG_I1i}'); return true" onMouseOut="info('&nbsp;'); return true"
> ${LANG_I62}

View File

@@ -129,15 +129,21 @@ ${do:copy:NoPwdInPages:hidepwd}
${do:copy:NoQueryStrings:hidequery}
${do:copy:NoPurgeOldFiles:nopurge}
${do:copy:Cookies:cookies}
${do:copy:CookiesFile:cookiesfile}
${do:copy:CheckType:checktype}
${do:copy:ParseJava:parsejava}
${do:copy:HTTP10:http10}
${do:copy:TolerantRequests:toler}
${do:copy:UpdateHack:updhack}
${do:copy:URLHack:urlhack}
${do:copy:KeepWww:keepwww}
${do:copy:KeepSlashes:keepslashes}
${do:copy:KeepQueryOrder:keepqueryorder}
${do:copy:StripQuery:stripquery}
${do:copy:StoreAllInCache:cache2}
${do:copy:LogType:logtype}
${do:copy:UseHTTPProxyForFTP:ftpprox}
${do:copy:ProxyType:proxytype}
${do:copy:Build:build}
${do:copy:PrimaryScan:filter}
${do:copy:Travel:travel}
@@ -148,6 +154,7 @@ ${do:copy:MaxHtml:maxhtml}
${do:copy:MaxOther:othermax}
${do:copy:MaxAll:sizemax}
${do:copy:MaxWait:pausebytes}
${do:copy:PauseFiles:pausefiles}
${do:copy:Sockets:connexion}
${do:copy:Retry:retry}
${do:copy:MaxTime:maxtime}

View File

@@ -178,6 +178,12 @@ ${do:end-if}
${test:parsejava:--parse-java=0:}
${test:updhack:--updatehack}
${test:urlhack:--urlhack=0:--urlhack}
${test:keepwww:--keep-www-prefix}
${test:keepslashes:--keep-double-slashes}
${test:keepqueryorder:--keep-query-order}
${test:cookiesfile:--cookies-file "}${html:cookiesfile}${test:cookiesfile:"}
${test:pausefiles:--pause "}${pausefiles}${test:pausefiles:"}
${test:stripquery:--strip-query "}${html:stripquery}${test:stripquery:"}
${test:toler:--tolerant}
${test:http10:--http-10}
${test:cache2:--store-all-in-cache}
@@ -186,7 +192,7 @@ ${do:end-if}
${test:logtype:::--extra-log:--debug-log}
${test:index:--index=0:}
${test:index2:--search-index=0:--search-index}
${test:prox:--proxy "}${prox}${test:prox:\3A}${portprox}${test:prox:"}
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype:socks5\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
</textarea>
@@ -219,15 +225,21 @@ NoPwdInPages=${ztest:hidepwd:0:1}
NoQueryStrings=${ztest:hidequery:0:1}
NoPurgeOldFiles=${ztest:nopurge:0:1}
Cookies=${ztest:cookies:0:1}
CookiesFile=${cookiesfile}
CheckType=${ztest:checktype:0:1:2}
ParseJava=${ztest:parsejava:0:1}
HTTP10=${ztest:http10:0:1}
TolerantRequests=${ztest:toler:0:1}
UpdateHack=${ztest:updhack:0:1}
URLHack=${ztest:urlhack:0:1}
KeepWww=${ztest:keepwww:0:1}
KeepSlashes=${ztest:keepslashes:0:1}
KeepQueryOrder=${ztest:keepqueryorder:0:1}
StripQuery=${stripquery}
StoreAllInCache=${ztest:cache2:0:1}
LogType=${logtype}
UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
ProxyType=${proxytype}
Build=${build}
PrimaryScan=${filter}
Travel=${travel}
@@ -238,6 +250,7 @@ MaxHtml=${maxhtml}
MaxOther=${othermax}
MaxAll=${sizemax}
MaxWait=${pausebytes}
PauseFiles=${pausefiles}
Sockets=${connexion}
Retry=${retry}
MaxTime=${maxtime}

View File

@@ -154,12 +154,14 @@ static const char *strjoker_rec(strjoker_memo *memo, const char *chaine,
if (depth > memo->maxdepth)
memo->maxdepth = depth;
/* Charge the budget before the depth cap: cut disables the memo, so uncounted
deep calls would let the sub-cap search explode (OSS-Fuzz 535114376). */
if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
return NULL; /* work budget spent: fail the match safely */
if (depth >= STRJOKER_MAXDEPTH) {
memo->cut = HTS_TRUE;
return NULL; /* nesting beyond any real filter: fail the branch safely */
}
if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
return NULL; /* work budget spent: fail the match safely */
if (memo->failed) {
bit = (size_t) (chaine - memo->chaine0) * memo->stride +
(size_t) (joker - memo->joker0);

View File

@@ -794,6 +794,20 @@ static int st_filterbounds(httrackp *opt, int argc, char **argv) {
memset(subj, 'a', 32);
subj[32] = '\0';
assertf(strjoker(subj, pat, NULL, NULL) != NULL);
/* Same pin for the class-branch shape users actually write (*[..]), against a
long subject: it must match with room to spare under the work cap. */
{
const char *seg = "*[A-Z,a-z,0-9]";
const size_t seglen = strlen(seg), nseg = 16;
for (i = 0; i < (int) nseg; i++)
memcpy(pat + i * seglen, seg, seglen);
pat[nseg * seglen] = '\0';
memset(subj, 'a', 512);
subj[512] = '\0';
assertf(strjoker_bounds(subj, pat, &steps, &maxsteps, NULL, NULL) != NULL);
assertf(steps < maxsteps);
}
freet(pat);
freet(subj);
printf("filterbounds: OK\n");