mirror of
https://github.com/xroche/httrack.git
synced 2026-07-27 02:52:42 +03:00
Compare commits
48 Commits
warnfix-p0
...
feat/singl
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b72e3dddf | ||
|
|
9484b32ecd | ||
|
|
1d647bfecd | ||
|
|
5ee188ba1a | ||
|
|
b2dc012263 | ||
|
|
d3a915641f | ||
|
|
aa1131982b | ||
|
|
15e2c701a5 | ||
|
|
6bd4209c94 | ||
|
|
f72e7ebe96 | ||
|
|
a228cede8a | ||
|
|
b5db9bb7ed | ||
|
|
3b53bf85f2 | ||
|
|
37ee129810 | ||
|
|
ce7dcfa9de | ||
|
|
069573edc3 | ||
|
|
a75f437df9 | ||
|
|
06dc54fa44 | ||
|
|
783f6ee1f5 | ||
|
|
913caf68be | ||
|
|
59660102d6 | ||
|
|
e96399910b | ||
|
|
52d0ab2356 | ||
|
|
42072f54b0 | ||
|
|
5f3787eb28 | ||
|
|
ca533abefd | ||
|
|
dc6819b7b7 | ||
|
|
4a62cfb75b | ||
|
|
906906efa7 | ||
|
|
f50307d878 | ||
|
|
c6b7c407ed | ||
|
|
3247e5b183 | ||
|
|
a1226401e0 | ||
|
|
71ca31e2ec | ||
|
|
b90d699d2e | ||
|
|
bef7120423 | ||
|
|
7cf54af485 | ||
|
|
c32a47110e | ||
|
|
1027a9f392 | ||
|
|
d0a1573a04 | ||
|
|
4362ac34cb | ||
|
|
6579436607 | ||
|
|
cb98b1b197 | ||
|
|
da8fbfff49 | ||
|
|
a7fbd3f739 | ||
|
|
c4b803eb33 | ||
|
|
408316db99 | ||
|
|
450db10d92 |
7
.github/workflows/windows-build.yml
vendored
7
.github/workflows/windows-build.yml
vendored
@@ -225,8 +225,11 @@ jobs:
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test"
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
|
||||
# webdav-mime needs a reapable background listener, which MSYS cannot give it;
|
||||
# badmtime needs a filesystem that stores an mtime past gmtime's range;
|
||||
# single-file ends on a GUI half needing htsserver, which this job does not build.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 93_local-single-file.test"
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
27
AGENTS.md
27
AGENTS.md
@@ -26,6 +26,12 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
check`, or `PATH="<bld>/src:$PATH"` for a manual run.
|
||||
- Give new `.test` scripts `set -e`: the older ones predate the rule, so several
|
||||
`local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
|
||||
- Never assert with `cmd | grep -q MARKER && fail`. Under `pipefail` the
|
||||
pipeline is non-zero both when `cmd` fails and when `grep -q` matches early
|
||||
and SIGPIPEs it, so the `&&` never fires and a probe that proved nothing reads
|
||||
as "marker absent". Capture the reply, assert the status line it must carry
|
||||
(an empty, truncated or redirected one is marker-free too), then match with a
|
||||
here-string.
|
||||
|
||||
## Hard invariants
|
||||
- **Generated autotools files are NOT in git.** `configure`, every
|
||||
@@ -48,6 +54,16 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
- Bounds-check every copy. Overflow-safe form: put the untrusted value alone,
|
||||
`untrusted < limit - controlled` — never `controlled + untrusted < limit`,
|
||||
which can wrap and pass.
|
||||
- **Abort or clip is a decision, not a default.** The `*_safe_` helpers
|
||||
(`strcpybuff`, `strlcpybuff`, `strcatbuff`) **abort** on overflow. Right for
|
||||
our own data, wrong for anything read back from a cache, a header or the
|
||||
wire, where it trades a memory smash for a crash on malformed input. Clip
|
||||
with `dst[0] = '\0'; strlncatbuff(dst, src, size, size - 1)`.
|
||||
- **A warning class is not the unsafe set.** `-Wformat-truncation` fires only on
|
||||
a *bounded* `snprintf` whose return is discarded, so an unbounded `sprintf`
|
||||
into the same buffer never appears on it. Before scoping a hardening pass off
|
||||
compiler output, grep the unguarded forms yourself (`\bsprintf\s*\(`,
|
||||
`\bstrcpy\s*\(`, `\bstrcat\s*\(`).
|
||||
|
||||
## C conventions
|
||||
- **Use the `*t` allocator wrappers, never raw libc** (`htssafe.h`):
|
||||
@@ -84,6 +100,17 @@ Before pushing, and when reviewing others, don't skim for bugs:
|
||||
layout/ABI, cache/wire format, or a security path? A static or unit check
|
||||
isn't enough; exercise the wrong behavior at runtime. Claude Code:
|
||||
`/review-recipe`.
|
||||
- **Poison a canary, never compare it against zero.** Checking that a
|
||||
neighbouring field is still `'\0'` cannot see the stray NUL an off-by-one
|
||||
terminator writes — the exact bug the canary is there for. Fill it with a
|
||||
non-zero byte, and prove it by killing both the stray-`'X'` and the
|
||||
stray-NUL mutant. Neither ASan nor `_FORTIFY_SOURCE` sees an overflow that
|
||||
lands inside the same struct.
|
||||
- **Overshoot every destination, not one.** A bounds test that oversizes a
|
||||
single field cannot tell a per-field bound from a one-size-fits-all one, nor
|
||||
from a fix that bounds that field and leaves its neighbours raw. Exercise
|
||||
each destination the path touches, spanning at least two capacities, and
|
||||
check what the code actually emits before writing the expected values.
|
||||
|
||||
## Commits
|
||||
- **Sign-off is mandatory.** Every commit carries a `Signed-off-by` trailer:
|
||||
|
||||
46
configure.ac
46
configure.ac
@@ -66,10 +66,11 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
|
||||
AC_SUBST(VERSION_INFO)
|
||||
|
||||
### Default CFLAGS
|
||||
# No -Wdeclaration-after-statement: nothing sets -std=, so this builds as gnu17.
|
||||
DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
|
||||
-Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
|
||||
-Wstrict-prototypes -Wmissing-prototypes \
|
||||
-Wmissing-declarations -Wdeclaration-after-statement \
|
||||
-Wmissing-declarations \
|
||||
-Wpointer-arith -Wsequence-point -Wnested-externs \
|
||||
-D_REENTRANT"
|
||||
AC_SUBST(DEFAULT_CFLAGS)
|
||||
@@ -77,27 +78,30 @@ DEFAULT_LDFLAGS=""
|
||||
AC_SUBST(DEFAULT_LDFLAGS)
|
||||
|
||||
### Additional flags (if supported)
|
||||
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"])
|
||||
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"])
|
||||
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"])
|
||||
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"])
|
||||
# -Werror on probes: exit-status-only checks let clang's warn-on-unknown-flag through.
|
||||
AX_CHECK_COMPILE_FLAG([-Wparentheses], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wparentheses"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Winit-self], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Winit-self"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wunused-but-set-parameter], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wunused-but-set-parameter"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Waddress], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Waddress"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wuninitialized], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wuninitialized"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat=2], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat=2"], [], [-Werror])
|
||||
# -Wformat-nonliteral needs -Wformat in the probe or gcc rejects it as ignored.
|
||||
AX_CHECK_COMPILE_FLAG([-Wformat-nonliteral], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wformat-nonliteral"], [], [-Werror -Wformat])
|
||||
AX_CHECK_COMPILE_FLAG([-Wmissing-parameter-type], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wmissing-parameter-type"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wold-style-definition], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wold-style-definition"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Wignored-qualifiers"], [], [-Werror])
|
||||
# Make htssafe.h's pointer-dest 'warning' attribute a hard error in our build
|
||||
# (migration is at zero; a new char* dest is a regression). gcc/clang each take
|
||||
# only their own spelling; downstream keeps the plain warning, not a build break.
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
|
||||
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
|
||||
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
|
||||
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
|
||||
[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"], [], [-Werror])], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"], [], [-Werror])
|
||||
AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"], [], [-Werror])
|
||||
# No --discard-all: it drops the local symbols naming every static function, so
|
||||
# a trace misattributes them to the nearest surviving global. Costs 0.6% size.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--no-undefined"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
|
||||
AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
|
||||
@@ -119,13 +123,13 @@ AC_SUBST([LIBC_FORCE_LINK])
|
||||
### PIE
|
||||
CFLAGS_PIE=""
|
||||
LDFLAGS_PIE=""
|
||||
AX_CHECK_COMPILE_FLAG([-fpie -pie], [CFLAGS_PIE="-fpie -pie"])
|
||||
AX_CHECK_COMPILE_FLAG([-fpie], [CFLAGS_PIE="-fpie"], [], [-Werror])
|
||||
AX_CHECK_LINK_FLAG([-pie], [LDFLAGS_PIE="-pie"])
|
||||
AC_SUBST([CFLAGS_PIE])
|
||||
AC_SUBST([LDFLAGS_PIE])
|
||||
|
||||
## Export all symbols for backtraces
|
||||
AX_CHECK_COMPILE_FLAG([-rdynamic], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -rdynamic"])
|
||||
# Ties a crash trace from a stripped build back to its separate debug symbols.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--build-id], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--build-id"])
|
||||
|
||||
### Check for -fvisibility=hidden support
|
||||
gl_VISIBILITY
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
if FUZZERS
|
||||
noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
|
||||
fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx \
|
||||
fuzz-htsparse
|
||||
fuzz-htsparse fuzz-singlefile
|
||||
endif
|
||||
|
||||
AM_CPPFLAGS = \
|
||||
@@ -27,6 +27,7 @@ fuzz_url_SOURCES = fuzz-url.c fuzz.h
|
||||
fuzz_header_SOURCES = fuzz-header.c fuzz.h
|
||||
fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
|
||||
fuzz_htsparse_SOURCES = fuzz-htsparse.c fuzz.h
|
||||
fuzz_singlefile_SOURCES = fuzz-singlefile.c fuzz.h
|
||||
|
||||
# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
|
||||
EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
@@ -47,4 +48,8 @@ EXTRA_DIST = README.md run-fuzzers.sh \
|
||||
corpus/cachendx/regress-overadvance.bin \
|
||||
corpus/cachendx/regress-truncated-entry.bin \
|
||||
corpus/htsparse/basic.html corpus/htsparse/script-inscript.html \
|
||||
corpus/htsparse/meta-usemap.html corpus/htsparse/malformed.html
|
||||
corpus/htsparse/meta-usemap.html corpus/htsparse/malformed.html \
|
||||
corpus/singlefile/img-src.html corpus/singlefile/link-rel.html \
|
||||
corpus/singlefile/style-block.html corpus/singlefile/style-attr.html \
|
||||
corpus/singlefile/srcset.html corpus/singlefile/rawtext.html \
|
||||
corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html
|
||||
|
||||
3
fuzz/corpus/singlefile/img-src.html
Normal file
3
fuzz/corpus/singlefile/img-src.html
Normal file
@@ -0,0 +1,3 @@
|
||||
<img src="a.png">
|
||||
<img src=big.png alt=over-cap>
|
||||
<img src="../escape.png"><img src="/abs.png"><img src="data:,x">
|
||||
4
fuzz/corpus/singlefile/link-rel.html
Normal file
4
fuzz/corpus/singlefile/link-rel.html
Normal file
@@ -0,0 +1,4 @@
|
||||
<link rel="stylesheet" href="s.css">
|
||||
<link rel=icon href=a.png>
|
||||
<link rel="next" href="p2.html">
|
||||
<link rel="preload" href="j.js">
|
||||
5
fuzz/corpus/singlefile/malformed.html
Normal file
5
fuzz/corpus/singlefile/malformed.html
Normal file
@@ -0,0 +1,5 @@
|
||||
<img src="unterminated.png
|
||||
<div style="background:url(a.png">
|
||||
<style>@import url(
|
||||
<!-- unterminated comment
|
||||
<a href=
|
||||
1
fuzz/corpus/singlefile/many-attrs.html
Normal file
1
fuzz/corpus/singlefile/many-attrs.html
Normal file
@@ -0,0 +1 @@
|
||||
<img src="a.png" a0="v" a1="v" a2="v" a3="v" a4="v" a5="v" a6="v" a7="v" a8="v" a9="v" a10="v" a11="v" a12="v" a13="v" a14="v" a15="v" a16="v" a17="v" a18="v" a19="v" a20="v" a21="v" a22="v" a23="v" a24="v" a25="v" a26="v" a27="v" a28="v" a29="v" a30="v" a31="v" a32="v" a33="v" a34="v" a35="v" a36="v" a37="v" a38="v" a39="v" a40="v" a41="v" a42="v" a43="v" a44="v" a45="v" a46="v" a47="v" a48="v" a49="v" a50="v" a51="v" a52="v" a53="v" a54="v" a55="v" a56="v" a57="v" a58="v" a59="v" a60="v" a61="v" a62="v" a63="v" a64="v" a65="v" a66="v" a67="v" a68="v" a69="v">
|
||||
4
fuzz/corpus/singlefile/rawtext.html
Normal file
4
fuzz/corpus/singlefile/rawtext.html
Normal file
@@ -0,0 +1,4 @@
|
||||
<script>var s="</scripting>"; if(a</b) x=1;</script>
|
||||
<script src="j.js"></script>
|
||||
<textarea></textareas></textarea>
|
||||
<title></titles></title>
|
||||
2
fuzz/corpus/singlefile/srcset.html
Normal file
2
fuzz/corpus/singlefile/srcset.html
Normal file
@@ -0,0 +1,2 @@
|
||||
<img srcset="a.png 1x, big.png 2x, a.png 100w">
|
||||
<source srcset="a.png,, a.png 2x," src="a.png">
|
||||
2
fuzz/corpus/singlefile/style-attr.html
Normal file
2
fuzz/corpus/singlefile/style-attr.html
Normal file
@@ -0,0 +1,2 @@
|
||||
<div style="background:url(a.png);list-style:url('a.png')"></div>
|
||||
<p style='background:url("a.png")'>x</p>
|
||||
5
fuzz/corpus/singlefile/style-block.html
Normal file
5
fuzz/corpus/singlefile/style-block.html
Normal file
@@ -0,0 +1,5 @@
|
||||
<style>@import "s.css";
|
||||
@import url(sub/b.css);
|
||||
div{background:url(a.png)}
|
||||
/* url(a.png) */ p:after{content:"url(a.png)"}
|
||||
</style>
|
||||
131
fuzz/fuzz-singlefile.c
Normal file
131
fuzz/fuzz-singlefile.c
Normal file
@@ -0,0 +1,131 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* Fuzz the --single-file rewriter (htssinglefile.c): hostile HTML walked
|
||||
through the tag, CSS url()/@import and srcset parsers, then re-serialized.
|
||||
The resolver is aimed at a private temp tree, so the inlining half (MIME
|
||||
guess, base64, nested stylesheet) is reached and nothing else on disk is. */
|
||||
#include "fuzz.h"
|
||||
|
||||
#include "httrack-library.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* Between a.png and big.png, so one input reaches both the inline path and the
|
||||
over-cap fallback. */
|
||||
#define FUZZ_SF_CAP 64
|
||||
|
||||
static char sf_root[512];
|
||||
static char sf_page[600];
|
||||
|
||||
/* The asset tree, in removal order: the subdirectory comes after its file. */
|
||||
static const char *const sf_files[] = {"a.png", "big.png", "j.js", "s.css",
|
||||
"sub/b.css", "sub", NULL};
|
||||
|
||||
static void sf_cleanup(void) {
|
||||
char path[700];
|
||||
int i;
|
||||
|
||||
for (i = 0; sf_files[i] != NULL; i++) {
|
||||
snprintf(path, sizeof(path), "%s/%s", sf_root, sf_files[i]);
|
||||
(void) remove(path);
|
||||
}
|
||||
(void) remove(sf_root);
|
||||
}
|
||||
|
||||
/* A missing asset would silently reduce the target to its parser half. */
|
||||
static void sf_write(const char *name, const char *data, size_t len) {
|
||||
char path[700];
|
||||
FILE *fp;
|
||||
|
||||
snprintf(path, sizeof(path), "%s/%s", sf_root, name);
|
||||
fp = fopen(path, "wb");
|
||||
if (fp == NULL || fwrite(data, 1, len, fp) != len)
|
||||
abort();
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
static void sf_text(const char *name, const char *data) {
|
||||
sf_write(name, data, strlen(data));
|
||||
}
|
||||
|
||||
static void sf_init(void) {
|
||||
static const char png[] = "\x89PNG\r\n\x1a\n";
|
||||
static const char big[4096] = "\x89PNG";
|
||||
const char *tmp = getenv("TMPDIR");
|
||||
char path[700];
|
||||
|
||||
hts_init();
|
||||
snprintf(sf_root, sizeof(sf_root), "%s/httrack-fuzz-sf-XXXXXX",
|
||||
tmp != NULL && tmp[0] != '\0' ? tmp : "/tmp");
|
||||
if (mkdtemp(sf_root) == NULL)
|
||||
abort();
|
||||
atexit(sf_cleanup);
|
||||
snprintf(sf_page, sizeof(sf_page), "%s/page.html", sf_root);
|
||||
snprintf(path, sizeof(path), "%s/sub", sf_root);
|
||||
if (mkdir(path, 0700) != 0)
|
||||
abort();
|
||||
sf_write("a.png", png, sizeof(png) - 1);
|
||||
sf_write("big.png", big, sizeof(big));
|
||||
sf_text("j.js", "var x=1;\n");
|
||||
/* @import plus a url(), so an inlined stylesheet recurses and its own
|
||||
relative reference is rebased. */
|
||||
sf_text("s.css", "@import url(sub/b.css);\ndiv{background:url(a.png)}\n");
|
||||
sf_text("sub/b.css", "p{background:url(../a.png)}\n");
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
static int inited = 0;
|
||||
|
||||
String out = STRING_EMPTY;
|
||||
httrackp *opt;
|
||||
/* Exact-length, unterminated: the rewriter is span-based, so ASan bounds a
|
||||
read past html_len instead of it landing on a terminator. */
|
||||
char *html = malloct(size != 0 ? size : 1);
|
||||
|
||||
if (!inited) {
|
||||
sf_init();
|
||||
inited = 1;
|
||||
}
|
||||
memcpy(html, data, size);
|
||||
|
||||
opt = hts_create_opt();
|
||||
opt->log = opt->errlog = NULL;
|
||||
opt->single_file_max_size = FUZZ_SF_CAP;
|
||||
|
||||
StringClear(out);
|
||||
(void) singlefile_rewrite_html(opt, sf_root, sf_page, html, size,
|
||||
SINGLEFILE_MAX_PAGE_SIZE, &out);
|
||||
|
||||
StringFree(out);
|
||||
freet(html);
|
||||
hts_free_opt(opt);
|
||||
return 0;
|
||||
}
|
||||
@@ -300,6 +300,7 @@ ones it kept so the local copy browses offline. These options tune both halves.<
|
||||
<tr><td><tt>--preserve (-%p), --disable-passwords (-%x)</tt></td><td>Leave HTML untouched (no rewriting), and strip passwords out of saved links.</td></tr>
|
||||
<tr><td><tt>--extended-parsing (-%P), --parse-java (-j)</tt></td><td>Aggressive link discovery, and how much script content is parsed for links.</td></tr>
|
||||
<tr><td><tt>--mime-html (-%M)</tt></td><td>Save the whole mirror as a single MIME-encapsulated <tt>.mht</tt> archive (<tt>index.mht</tt>).</td></tr>
|
||||
<tr><td><tt>--single-file (-%Z), --single-file-max-size N</tt></td><td>Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded as <tt>data:</tt> URIs. Assets over the cap (10 MB by default) keep their link, as do audio, video, and the links from one page to another. A sibling of <tt>-%M</tt>, not a replacement: see the recipe below for which to pick.</td></tr>
|
||||
<tr><td><tt>--index (-I), --build-top-index (-%i), --search-index (-%I)</tt></td><td>Build a per-mirror index, a top index across projects, and a searchable keyword index.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -482,6 +483,25 @@ add <tt>--warc-cdx</tt> for a sorted CDXJ index, or <tt>--wacz</tt> to bundle th
|
||||
archive, index and pages into one WACZ for replay tools such as
|
||||
replayweb.page.</small></p>
|
||||
|
||||
<h4>Pages you can hand to someone as one file</h4>
|
||||
<p><tt>httrack https://example.com/ --single-file --path mydir</tt><br>
|
||||
<small>Rewrites each saved page after the crawl so its stylesheets, scripts,
|
||||
images and fonts are embedded as <tt>data:</tt> URIs. The mirror stays a normal
|
||||
browsable tree, with links between pages relative and the assets still on disk,
|
||||
but any single <tt>.html</tt> file can now be mailed or archived on its own.
|
||||
Raise or lower the 10 MB per-asset limit with
|
||||
<tt>--single-file-max-size N</tt>; anything over it, plus audio and video, keeps
|
||||
its link. One caveat if you raise it: an inlined stylesheet becomes a
|
||||
<tt>data:</tt> URL, whose path is opaque, so an asset it referenced that stayed
|
||||
over the cap no longer resolves from inside it. Raising the cap past that asset
|
||||
embeds it too and the question goes away.<br>
|
||||
Reach for this when the file has to open for someone you cannot make assumptions
|
||||
about: it is plain HTML and needs no add-on. Reach for <tt>--mime-html</tt>
|
||||
instead when a Chromium-family browser is a given and the mirror is large: MIME
|
||||
carries text parts without the base64 tax, keeps each resource's original URL,
|
||||
and stores a shared asset once rather than re-embedding it in every page that
|
||||
uses it.</small></p>
|
||||
|
||||
<h4>HTTrack as a fetch tool</h4>
|
||||
<p><tt>httrack --get https://host/file.bin --path tmp</tt><br>
|
||||
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
|
||||
|
||||
@@ -90,9 +90,10 @@ offline browser : copy websites to a local directory</p>
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
<b>-%x, --disable-passwords</b> ] [ <b>-%q,
|
||||
] [ <b>-%Z, --single-file</b> ] [ <b>-LN,
|
||||
--long-names[=N]</b> ] [ <b>-KN, --keep-links[=N]</b> ] [
|
||||
<b>-x, --replace-external</b> ] [ <b>-%x,
|
||||
--disable-passwords</b> ] [ <b>-%q,
|
||||
--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
|
||||
<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
|
||||
] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
|
||||
@@ -648,6 +649,24 @@ don’t wait) (--cached-delayed-type-check)</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%Z</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>after the mirror, rewrite each saved page with its
|
||||
stylesheets, scripts, images and fonts inlined as data:
|
||||
URIs, so any page opens by double-click anywhere (links
|
||||
between pages stay relative; audio and video stay links);
|
||||
--single-file-max-size N caps each asset (default 10485760
|
||||
bytes). %M is the better container where a Chromium-family
|
||||
browser is a given: one archive, no base64 tax on text, a
|
||||
shared asset stored once (--single-file)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%t</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
@@ -97,9 +97,8 @@ ${do:end-if}
|
||||
</pre>
|
||||
|
||||
${LANG_G8} :
|
||||
${do:output-mode:html-urlescaped}
|
||||
<a href="file://${path}/${projname}/" target="_new">
|
||||
${do:output-mode:}
|
||||
${/* an http: page cannot navigate to file:, so the mirror is reached through the server */}
|
||||
<a href="/website/index.html" target="_new">
|
||||
${path}/${projname}
|
||||
</a></li>
|
||||
<ul>
|
||||
|
||||
@@ -98,6 +98,9 @@ ${do:end-if}
|
||||
<input type="hidden" name="redirect" value="">
|
||||
<input type="hidden" name="closeme" value="">
|
||||
|
||||
<!-- clear if not checked -->
|
||||
<input type="hidden" name="ftpprox" value="">
|
||||
|
||||
${LANG_PROXYTYPE}
|
||||
<select name="proxytype"
|
||||
title='${html:LANG_PROXYTYPETIP}' onMouseOver="info('${html:LANG_PROXYTYPETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
|
||||
@@ -98,6 +98,14 @@ ${do:end-if}
|
||||
<input type="hidden" name="redirect" value="">
|
||||
<input type="hidden" name="closeme" value="">
|
||||
|
||||
<!-- clear if not checked -->
|
||||
<input type="hidden" name="errpage" value="">
|
||||
<input type="hidden" name="external" value="">
|
||||
<input type="hidden" name="hidepwd" value="">
|
||||
<input type="hidden" name="hidequery" value="">
|
||||
<input type="hidden" name="nopurge" value="">
|
||||
<input type="hidden" name="singlefile" value="">
|
||||
|
||||
${LANG_I33}
|
||||
<br>
|
||||
<select name="build"
|
||||
@@ -136,6 +144,13 @@ ${listid:build:LISTDEF_3}
|
||||
<tr><td><input type="checkbox" name="nopurge" ${checked:nopurge}
|
||||
title='${html:LANG_I1a}' onMouseOver="info('${html:LANG_I1a}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I57}</td></tr>
|
||||
<tr><td><input type="checkbox" name="singlefile" ${checked:singlefile}
|
||||
title='${html:LANG_SINGLEFILETIP}' onMouseOver="info('${html:LANG_SINGLEFILETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_SINGLEFILE}</td></tr>
|
||||
<tr><td>${LANG_SINGLEFILEMAX}
|
||||
<input name="singlefilemax" value="${singlefilemax}" size="12"
|
||||
title='${html:LANG_SINGLEFILEMAXTIP}' onMouseOver="info('${html:LANG_SINGLEFILEMAXTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
></td></tr>
|
||||
</table>
|
||||
|
||||
<tr><td>
|
||||
|
||||
@@ -103,7 +103,7 @@ ${LANG_Q3}
|
||||
<tr><td>
|
||||
<table width="100%">
|
||||
<tr><td align="left">
|
||||
<input type="submit" value="${LANG_OK]"
|
||||
<input type="submit" value="${LANG_OK}"
|
||||
${do:output-mode:html-urlescaped}
|
||||
onClick="if (confirm(str_replace(str_replace('${LANG_DIAL7}', '%20', ' '), '%0a', ' '))) { form.closeme.value=1; form.submit(); } return false;"
|
||||
${do:output-mode:}
|
||||
|
||||
@@ -98,6 +98,9 @@ ${do:end-if}
|
||||
<input type="hidden" name="redirect" value="">
|
||||
<input type="hidden" name="closeme" value="">
|
||||
|
||||
<!-- clear if not checked -->
|
||||
<input type="hidden" name="windebug" value="">
|
||||
|
||||
${LANG_I40c}
|
||||
<br>
|
||||
|
||||
|
||||
@@ -98,6 +98,11 @@ ${do:end-if}
|
||||
<input type="hidden" name="redirect" value="">
|
||||
<input type="hidden" name="closeme" value="">
|
||||
|
||||
<!-- clear if not checked -->
|
||||
<input type="hidden" name="ka" value="">
|
||||
<input type="hidden" name="remt" value="">
|
||||
<input type="hidden" name="rems" value="">
|
||||
|
||||
<table border="0" width="100%" cellspacing="0">
|
||||
|
||||
<tr><td>
|
||||
|
||||
@@ -98,6 +98,17 @@ ${do:end-if}
|
||||
<input type="hidden" name="redirect" value="">
|
||||
<input type="hidden" name="closeme" value="">
|
||||
|
||||
<!-- clear if not checked -->
|
||||
<input type="hidden" name="cookies" value="">
|
||||
<input type="hidden" name="parsejava" value="">
|
||||
<input type="hidden" name="updhack" value="">
|
||||
<input type="hidden" name="urlhack" value="">
|
||||
<input type="hidden" name="keepwww" value="">
|
||||
<input type="hidden" name="keepslashes" value="">
|
||||
<input type="hidden" name="keepqueryorder" value="">
|
||||
<input type="hidden" name="toler" value="">
|
||||
<input type="hidden" name="http10" value="">
|
||||
|
||||
<input type="checkbox" name="cookies" ${checked:cookies}
|
||||
title='${html:LANG_I1b}' onMouseOver="info('${html:LANG_I1b}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I58}
|
||||
|
||||
@@ -98,6 +98,13 @@ ${do:end-if}
|
||||
<input type="hidden" name="redirect" value="">
|
||||
<input type="hidden" name="closeme" value="">
|
||||
|
||||
<!-- clear if not checked -->
|
||||
<input type="hidden" name="warc" value="">
|
||||
<input type="hidden" name="norecatch" value="">
|
||||
<input type="hidden" name="logf" value="">
|
||||
<input type="hidden" name="index" value="">
|
||||
<input type="hidden" name="index2" value="">
|
||||
|
||||
<input type="checkbox" name="cache2" ${checked:cache2}
|
||||
title='${html:LANG_I1e}' onMouseOver="info('${html:LANG_I1e}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I61}
|
||||
|
||||
@@ -143,6 +143,8 @@ ${do:copy:StripQuery:stripquery}
|
||||
${do:copy:StoreAllInCache:cache2}
|
||||
${do:copy:Warc:warc}
|
||||
${do:copy:WarcFile:warcfile}
|
||||
${do:copy:SingleFile:singlefile}
|
||||
${do:copy:SingleFileMaxSize:singlefilemax}
|
||||
${do:copy:LogType:logtype}
|
||||
${do:copy:UseHTTPProxyForFTP:ftpprox}
|
||||
${do:copy:ProxyType:proxytype}
|
||||
|
||||
@@ -114,16 +114,16 @@ ${do:end-if}
|
||||
|
||||
${/* Real commands and ini file generated below */}
|
||||
|
||||
<!-- engine commandline -->
|
||||
<!-- engine commandline; ztest so a cleared default-on option still emits its disabling flag -->
|
||||
${do:output-mode:html}
|
||||
<textarea name="command" cols="50" rows="4" style="visibility:hidden">
|
||||
httrack \
|
||||
--quiet \
|
||||
--build-top-index \
|
||||
${test:todo:--mirror:--mirror:--mirror-wizard:--get:--mirrorlinks:--testlinks:--continue:--update}
|
||||
${urls}
|
||||
${test:filelist:-%L "}${filelist}${test:filelist:"}
|
||||
--path "${html:path}/${html:projname}"
|
||||
${unquoted:urls}
|
||||
${test:filelist:-%L "}${arg:filelist}${test:filelist:"}
|
||||
--path "${arg:path}/${arg:projname}"
|
||||
\
|
||||
${test:parseall:--near}
|
||||
${test:link:--test}
|
||||
@@ -131,7 +131,7 @@ httrack \
|
||||
${test:htmlfirst::--priority=7}
|
||||
\
|
||||
${do:if-not-empty:BuildString}
|
||||
--structure "${BuildString}"
|
||||
--structure "${arg:BuildString}"
|
||||
${do:end-if}
|
||||
${test:build:-N0:-N0:-N1:-N2:-N3:-N4:-N5:-N100:-N101:-N102:-N103:-N104:-N105:-N99:-N199:}
|
||||
\
|
||||
@@ -150,51 +150,54 @@ ${do:end-if}
|
||||
${test:travel3::--keep-links=0:--keep-links:--keep-links=3:--keep-links=4}
|
||||
${test:windebug:--debug-headers}
|
||||
\
|
||||
${test:connexion:--sockets=}${connexion}
|
||||
${test:connexion:--sockets=}${unquoted:connexion}
|
||||
${test:ka:--keep-alive}
|
||||
${test:timeout:--timeout=}${timeout}
|
||||
${test:timeout:--timeout=}${unquoted:timeout}
|
||||
${test:remt:--host-control=1}
|
||||
${test:retry:--retries=}${retry}
|
||||
${test:rate:--min-rate=}${rate}
|
||||
${test:retry:--retries=}${unquoted:retry}
|
||||
${test:rate:--min-rate=}${unquoted:rate}
|
||||
${test:rems:--host-control=2}
|
||||
\
|
||||
${test:depth:--depth=}${depth}
|
||||
${test:depth2:--ext-depth=}${depth2}
|
||||
${test:maxhtml:--max-files=,}${maxhtml}
|
||||
${test:othermax:--max-files=}${othermax}
|
||||
${test:sizemax:--max-files=}${sizemax}
|
||||
${test:pausebytes:--max-pause=}${pausebytes}
|
||||
${test:maxtime:--max-time=}${maxtime}
|
||||
${test:maxrate:--max-rate=}${maxrate}
|
||||
${test:maxconn:--connection-per-second=}${maxconn}
|
||||
${test:maxlinks:--advanced-maxlinks=}${maxlinks}
|
||||
${test:depth:--depth=}${unquoted:depth}
|
||||
${test:depth2:--ext-depth=}${unquoted:depth2}
|
||||
${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one */}
|
||||
${test:othermax:--max-files=}${unquoted:othermax}
|
||||
${test:maxhtml:--max-files=,}${unquoted:maxhtml}
|
||||
${test:sizemax:--max-size=}${unquoted:sizemax}
|
||||
${test:pausebytes:--max-pause=}${unquoted:pausebytes}
|
||||
${test:maxtime:--max-time=}${unquoted:maxtime}
|
||||
${test:maxrate:--max-rate=}${unquoted:maxrate}
|
||||
${test:maxconn:--connection-per-second=}${unquoted:maxconn}
|
||||
${test:maxlinks:--advanced-maxlinks=}${unquoted:maxlinks}
|
||||
\
|
||||
--user-agent "${html:user}"
|
||||
--footer "${html:footer}"
|
||||
--user-agent "${arg:user}"
|
||||
--footer "${arg:footer}"
|
||||
\
|
||||
${url2}
|
||||
${unquoted:url2}
|
||||
\
|
||||
${test:cookies:--cookies=0:}
|
||||
${test:parsejava:--parse-java=0:}
|
||||
${ztest:cookies:--cookies=0:}
|
||||
${ztest:parsejava:--parse-java=0:}
|
||||
${test:updhack:--updatehack}
|
||||
${test:urlhack:--urlhack=0:--urlhack}
|
||||
${test:keepwww:--keep-www-prefix}
|
||||
${test:keepslashes:--keep-double-slashes}
|
||||
${test:keepqueryorder:--keep-query-order}
|
||||
${test:cookiesfile:--cookies-file "}${html:cookiesfile}${test:cookiesfile:"}
|
||||
${test:pausefiles:--pause "}${pausefiles}${test:pausefiles:"}
|
||||
${test:stripquery:--strip-query "}${html:stripquery}${test:stripquery:"}
|
||||
${test:cookiesfile:--cookies-file "}${arg:cookiesfile}${test:cookiesfile:"}
|
||||
${test:pausefiles:--pause "}${arg:pausefiles}${test:pausefiles:"}
|
||||
${test:stripquery:--strip-query "}${arg:stripquery}${test:stripquery:"}
|
||||
${test:toler:--tolerant}
|
||||
${test:http10:--http-10}
|
||||
${test:cache2:--store-all-in-cache}
|
||||
${test:warc:--warc}
|
||||
${test:warcfile:--warc-file "}${html:warcfile}${test:warcfile:"}
|
||||
${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
|
||||
${test:singlefile:--single-file}
|
||||
${test:singlefilemax:--single-file-max-size=}${unquoted:singlefilemax}
|
||||
${test:norecatch:--do-not-recatch}
|
||||
${test:logf:--single-log}
|
||||
${test:logtype:::--extra-log:--debug-log}
|
||||
${test:index:--index=0:}
|
||||
${test:index2:--search-index=0:--search-index}
|
||||
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
|
||||
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${arg:prox}${test:prox:\3A}${arg:portprox}${test:prox:"}
|
||||
${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
|
||||
</textarea>
|
||||
|
||||
@@ -213,7 +216,7 @@ ParseAll=${ztest:parseall:0:1}
|
||||
HTMLFirst=${ztest:htmlfirst:0:1}
|
||||
Cache=${ztest:cache:0:1}
|
||||
NoRecatch=${ztest:norecatch:0:1}
|
||||
Dos=${dos
|
||||
Dos=${dos}
|
||||
Index=${ztest:index:0:1}
|
||||
WordIndex=${ztest:index2:0:1}
|
||||
Log=${ztest:logf:0:1:2}
|
||||
@@ -241,6 +244,8 @@ StripQuery=${stripquery}
|
||||
StoreAllInCache=${ztest:cache2:0:1}
|
||||
Warc=${ztest:warc:0:1}
|
||||
WarcFile=${warcfile}
|
||||
SingleFile=${ztest:singlefile:0:1}
|
||||
SingleFileMaxSize=${singlefilemax}
|
||||
LogType=${logtype}
|
||||
UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
|
||||
ProxyType=${proxytype}
|
||||
|
||||
8
lang.def
8
lang.def
@@ -1042,3 +1042,11 @@ LANG_WARCFILE
|
||||
WARC archive name:
|
||||
LANG_WARCFILETIP
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
LANG_SINGLEFILE
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
LANG_SINGLEFILETIP
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
LANG_SINGLEFILEMAX
|
||||
Largest inlined asset (bytes):
|
||||
LANG_SINGLEFILEMAXTIP
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Èìå íà WARC àðõèâà:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Íåçàäúëæèòåëíî áàçîâî èìå çà WARC àðõèâà; îñòàâåòå ïðàçíî çà àâòîìàòè÷íî èìåíóâàíå â èçõîäíàòà äèðåêòîðèÿ.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Âãðàæäàíå íà ðåñóðñèòå êàòî data: URI (ñàìîñòîÿòåëíè ñòðàíèöè)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ñëåä çàâúðøâàíå íà îãëåäàëîòî âñÿêà çàïàçåíà ñòðàíèöà ñå ïðåçàïèñâà ñ âãðàäåíè ñòèëîâå, ñêðèïòîâå, èçîáðàæåíèÿ è øðèôòîâå, òàêà ÷å ñòðàíèöàòà äà ìîæå äà ñå îòâîðè è ñàìîñòîÿòåëíî.
|
||||
Largest inlined asset (bytes):
|
||||
Íàé-ãîëÿì âãðàäåí ðåñóðñ (áàéòîâå):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Ðåñóðñ íàä òîçè ðàçìåð çàïàçâà îáèêíîâåíà âðúçêà; îñòàâåòå ïðàçíî çà ñòîéíîñòòà ïî ïîäðàçáèðàíå îò 10485760 áàéòà.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Nombre del archivo WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nombre base opcional para el archivo WARC; déjelo en blanco para nombrarlo automáticamente en el directorio de salida.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incrustar los recursos como URI data: (páginas autónomas)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Una vez terminada la copia, reescribir cada página guardada con sus hojas de estilo, scripts, imágenes y fuentes incrustadas, de modo que una página también pueda abrirse por sí sola.
|
||||
Largest inlined asset (bytes):
|
||||
Tamaño máximo del recurso incrustado (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Un recurso mayor que este tamaño conserva un enlace normal; déjelo vacío para el valor predeterminado de 10485760 bytes.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Název archivu WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Volitelný základní název archivu WARC; ponechte prázdné pro automatické pojmenování ve výstupním adresáøi.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Vložit zdroje jako URI data: (samostatné stránky)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Po dokonèení zrcadlení pøepsat každou uloženou stránku s vloženými styly, skripty, obrázky a písmy, aby ji bylo možné otevøít i samostatnì.
|
||||
Largest inlined asset (bytes):
|
||||
Nejvìtší vložený zdroj (bajty):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Zdroj vìtší než tato velikost si ponechá bìžný odkaz; ponechte prázdné pro výchozí hodnotu 10485760 bajtù.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC 封存檔名稱:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC 封存檔的選用基本名稱;留空則於輸出目錄中自動命名。
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
將資源內嵌為 data: URI(獨立網頁)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
鏡射完成後,重寫每個已儲存的網頁,將樣式表、指令碼、圖片與字型內嵌其中,讓網頁也能單獨開啟。
|
||||
Largest inlined asset (bytes):
|
||||
內嵌資源大小上限(位元組):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
超過此大小的資源會保留一般連結;留空則使用預設的 10485760 位元組。
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC 归档名称:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC 归档的可选基本名称;留空则在输出目录中自动命名。
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
将资源内嵌为 data: URI(独立网页)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
镜像完成后,重写每个已保存的网页,将样式表、脚本、图片和字体内嵌其中,使网页也能单独打开。
|
||||
Largest inlined asset (bytes):
|
||||
内嵌资源大小上限(字节):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
超过此大小的资源会保留普通链接;留空则使用默认的 10485760 字节。
|
||||
|
||||
@@ -966,3 +966,11 @@ WARC archive name:
|
||||
Naziv WARC arhive:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Neobavezni osnovni naziv WARC arhive; ostavite prazno za automatsko imenovanje u izlaznom direktoriju.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Ugradi resurse kao data: URI-je (samostalne stranice)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Nakon dovr¹etka zrcaljenja ponovno zapi¹i svaku spremljenu stranicu s ugraðenim stilskim listovima, skriptama, slikama i fontovima, tako da se stranica mo¾e otvoriti i zasebno.
|
||||
Largest inlined asset (bytes):
|
||||
Najveæi ugraðeni resurs (bajtovi):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Resurs veæi od ove velièine zadr¾ava obiènu poveznicu; ostavite prazno za zadanih 10485760 bajtova.
|
||||
|
||||
@@ -1012,3 +1012,11 @@ WARC archive name:
|
||||
Navn på WARC-arkiv:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valgfrit basisnavn til WARC-arkivet; lad feltet stå tomt for automatisk navngivning i outputmappen.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Indlejr ressourcer som data:-URI'er (selvstændige sider)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Når spejlingen er færdig, omskrives hver gemt side med dens typografiark, scripts, billeder og skrifttyper indlejret, så en side også kan åbnes alene.
|
||||
Largest inlined asset (bytes):
|
||||
Største indlejrede ressource (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
En ressource over denne størrelse beholder et almindeligt link; lad feltet stå tomt for standardværdien på 10485760 byte.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Name des WARC-Archivs:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optionaler Basisname für das WARC-Archiv; leer lassen, um es automatisch im Ausgabeverzeichnis zu benennen.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Ressourcen als data:-URIs einbetten (eigenständige Seiten)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Nach Abschluss der Spiegelung jede gespeicherte Seite mit eingebetteten Stylesheets, Skripten, Bildern und Schriftarten neu schreiben, sodass eine Seite auch für sich allein geöffnet werden kann.
|
||||
Largest inlined asset (bytes):
|
||||
Größte eingebettete Ressource (Bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Eine Ressource über dieser Größe behält einen gewöhnlichen Link; leer lassen für den Standardwert von 10485760 Bytes.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC-arhiivi nimi:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC-arhiivi valikuline põhinimi; jäta tühjaks, et see väljundkataloogis automaatselt nimetada.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Manusta ressursid data: URI-dena (iseseisvad lehed)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Pärast peegelduse lõppu kirjuta iga salvestatud leht ümber, manustades selle laaditabelid, skriptid, pildid ja fondid, nii et lehte saab avada ka eraldi.
|
||||
Largest inlined asset (bytes):
|
||||
Suurim manustatud ressurss (baiti):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Sellest suurem ressurss säilitab tavalise lingi; jäta tühjaks vaikeväärtuse 10485760 baiti jaoks.
|
||||
|
||||
@@ -1012,3 +1012,11 @@ WARC archive name:
|
||||
WARC archive name:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Largest inlined asset (bytes):
|
||||
Largest inlined asset (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
|
||||
@@ -966,3 +966,11 @@ WARC archive name:
|
||||
WARC-arkiston nimi:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valinnainen WARC-arkiston perusnimi; jätä tyhjäksi, jotta se nimetään automaattisesti tulostehakemistoon.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Upota resurssit data:-URI-osoitteina (itsenäiset sivut)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Kun peilaus on valmis, kirjoita jokainen tallennettu sivu uudelleen niin, että sen tyylitiedostot, komentosarjat, kuvat ja fontit upotetaan, jolloin sivun voi avata myös yksinään.
|
||||
Largest inlined asset (bytes):
|
||||
Suurin upotettu resurssi (tavua):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Tätä suurempi resurssi säilyttää tavallisen linkin; jätä tyhjäksi, jolloin käytetään oletusarvoa 10485760 tavua.
|
||||
|
||||
@@ -1012,3 +1012,11 @@ WARC archive name:
|
||||
Nom de l'archive WARC :
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nom de base optionnel pour l'archive WARC ; laissez vide pour le générer automatiquement dans le répertoire de sortie.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Intégrer les ressources en URI data: (pages autonomes)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Une fois le miroir terminé, réécrire chaque page enregistrée avec ses feuilles de style, scripts, images et polices intégrés, afin qu'une page puisse aussi être ouverte seule.
|
||||
Largest inlined asset (bytes):
|
||||
Taille maximale d'une ressource intégrée (octets) :
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Au-delà de cette taille, la ressource reste un lien ordinaire ; laissez vide pour la valeur par défaut de 10485760 octets.
|
||||
|
||||
@@ -966,3 +966,11 @@ WARC archive name:
|
||||
Όνομα αρχείου WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Προαιρετικό βασικό όνομα για το αρχείο WARC. Αφήστε το κενό για αυτόματη ονομασία στον κατάλογο εξόδου.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Ενσωμάτωση των πόρων ως URI data: (αυτόνομες σελίδες)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Μόλις ολοκληρωθεί το αντίγραφο, κάθε αποθηκευμένη σελίδα ξαναγράφεται με ενσωματωμένα τα φύλλα στυλ, τα σενάρια, τις εικόνες και τις γραμματοσειρές της, ώστε η σελίδα να μπορεί να ανοίξει και μόνη της.
|
||||
Largest inlined asset (bytes):
|
||||
Μέγιστος ενσωματωμένος πόρος (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Πόρος μεγαλύτερος από αυτό το μέγεθος διατηρεί κανονικό σύνδεσμο. Αφήστε το κενό για την προεπιλογή των 10485760 byte.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Nome dell'archivio WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nome di base facoltativo per l'archivio WARC; lascia vuoto per assegnarlo automaticamente nella directory di output.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incorpora le risorse come URI data: (pagine autonome)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Al termine della copia, riscrive ogni pagina salvata incorporando fogli di stile, script, immagini e caratteri, in modo che una pagina possa essere aperta anche da sola.
|
||||
Largest inlined asset (bytes):
|
||||
Dimensione massima della risorsa incorporata (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Una risorsa oltre questa dimensione mantiene un collegamento normale; lasciare vuoto per il valore predefinito di 10485760 byte.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC アーカイブ名:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC アーカイブの任意のベース名。空欄にすると出力ディレクトリ内で自動的に名前が付けられます。
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
リソースを data: URI として埋め込む (単独で開けるページ)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
ミラーの完了後、保存された各ページを、スタイルシート、スクリプト、画像、フォントを埋め込んだ形で書き直します。ページ単体でも開けるようになります。
|
||||
Largest inlined asset (bytes):
|
||||
埋め込む最大サイズ (バイト):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
このサイズを超えるリソースは通常のリンクのままになります。空欄にすると既定値の 10485760 バイトになります。
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
¸ÜÕ ÝÐ WARC ÐàåØÒÐâÐ:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
¸×ÑÞàÝÞ ÞáÝÞÒÝÞ ØÜÕ ×Ð WARC ÐàåØÒÐâÐ; ÞáâÐÒÕâÕ ßàÐ×ÝÞ ×Ð ÐÒâÞÜÐâáÚÞ ØÜÕÝãÒÐúÕ ÒÞ Ø×ÛÕ×ÝØÞâ ÔØàÕÚâÞàØãÜ.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
²ÓàÐÔØ ÓØ àÕáãàáØâÕ ÚÐÚÞ data: URI (áÐÜÞáâÞøÝØ áâàÐÝØæØ)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
¿Þ ×ÐÒàèãÒÐúÕ ÝÐ ÞÓÛÕÔÐÛÞâÞ, áÕÚÞøÐ ×ÐçãÒÐÝÐ áâàÐÝØæÐ áÕ ßàÕߨèãÒÐ áÞ ÒÓàÐÔÕÝØ áâØÛáÚØ ÛØáâÞÒØ, áÚàØßâØ, áÛØÚØ Ø äÞÝâÞÒØ, ×Ð ÔÐ ÜÞÖÕ áâàÐÝØæÐâÐ ÔÐ áÕ ÞâÒÞàØ Ø áÐÜÞáâÞøÝÞ.
|
||||
Largest inlined asset (bytes):
|
||||
½ÐøÓÞÛÕÜ ÒÓàÐÔÕÝ àÕáãàá (ÑÐøâØ):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
ÀÕáãàá ßÞÓÞÛÕÜ ÞÔ ÞÒÐÐ ÓÞÛÕÜØÝÐ ×ÐÔàÖãÒÐ ÞÑØçÝÐ ÒàáÚÐ; ÞáâÐÒÕâÕ ßàÐ×ÝÞ ×Ð áâÐÝÔÐàÔÝØâÕ 10485760 ÑÐøâØ.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC archívum neve:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
A WARC archívum opcionális alapneve; hagyja üresen az automatikus elnevezéshez a kimeneti könyvtárban.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Erõforrások beágyazása data: URI-ként (önálló oldalak)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
A tükrözés befejezése után minden mentett oldal újraírása a stíluslapok, parancsfájlok, képek és betûtípusok beágyazásával, így az oldal önmagában is megnyitható.
|
||||
Largest inlined asset (bytes):
|
||||
Legnagyobb beágyazott erõforrás (bájt):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Az ennél nagyobb erõforrás közönséges hivatkozás marad; hagyja üresen a 10485760 bájtos alapértelmezéshez.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Naam van WARC-archief:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optionele basisnaam voor het WARC-archief; laat leeg om het automatisch een naam te geven in de uitvoermap.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Bronnen insluiten als data:-URI's (op zichzelf staande pagina's)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Zodra de spiegeling klaar is, elke opgeslagen pagina herschrijven met ingesloten stijlbladen, scripts, afbeeldingen en lettertypen, zodat een pagina ook los geopend kan worden.
|
||||
Largest inlined asset (bytes):
|
||||
Grootste ingesloten bron (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Een bron boven deze grootte houdt een gewone koppeling; laat leeg voor de standaardwaarde van 10485760 bytes.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Navn på WARC-arkiv:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valgfritt basisnavn for WARC-arkivet; la feltet stå tomt for automatisk navngivning i utdatamappen.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Bygg inn ressurser som data:-URI-er (selvstendige sider)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Når speilingen er ferdig, skrives hver lagrede side om med stilark, skript, bilder og skrifter innebygd, slik at en side også kan åpnes alene.
|
||||
Largest inlined asset (bytes):
|
||||
Største innebygde ressurs (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
En ressurs over denne størrelsen beholder en vanlig lenke; la feltet stå tomt for standardverdien på 10485760 byte.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Nazwa archiwum WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Opcjonalna nazwa bazowa archiwum WARC; pozostaw puste, aby nazwaæ je automatycznie w katalogu wyj¶ciowym.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Osad¼ zasoby jako identyfikatory data: URI (samodzielne strony)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Po zakoñczeniu tworzenia kopii przepisz ka¿d± zapisan± stronê z osadzonymi arkuszami stylów, skryptami, obrazami i czcionkami, aby stronê mo¿na by³o otworzyæ równie¿ samodzielnie.
|
||||
Largest inlined asset (bytes):
|
||||
Najwiêkszy osadzony zasób (bajty):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Zasób wiêkszy ni¿ ten rozmiar zachowuje zwyk³y odno¶nik; pozostaw puste, aby u¿yæ domy¶lnych 10485760 bajtów.
|
||||
|
||||
@@ -1012,3 +1012,11 @@ WARC archive name:
|
||||
Nome do arquivo WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nome base opcional para o arquivo WARC; deixe em branco para nomeá-lo automaticamente no diretório de saída.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incorporar os recursos como URIs data: (páginas autônomas)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ao terminar o espelhamento, reescrever cada página salva com suas folhas de estilo, scripts, imagens e fontes incorporadas, para que a página também possa ser aberta sozinha.
|
||||
Largest inlined asset (bytes):
|
||||
Maior recurso incorporado (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Um recurso acima desse tamanho mantém um link comum; deixe em branco para o padrão de 10485760 bytes.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Nome do arquivo WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nome base opcional para o arquivo WARC; deixe em branco para o nomear automaticamente no diretório de saída.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Incorporar os recursos como URI data: (páginas autónomas)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Depois de terminado o espelho, reescrever cada página guardada com as suas folhas de estilo, scripts, imagens e tipos de letra incorporados, para que a página também possa ser aberta sozinha.
|
||||
Largest inlined asset (bytes):
|
||||
Maior recurso incorporado (bytes):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Um recurso acima deste tamanho mantém uma ligação normal; deixe em branco para o valor predefinido de 10485760 bytes.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Numele arhivei WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nume de baza optional pentru arhiva WARC; lasati gol pentru a-l denumi automat in directorul de iesire.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Încorporeaza resursele ca URI data: (pagini de sine statatoare)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Dupa terminarea copiei, fiecare pagina salvata este rescrisa cu foile de stil, scripturile, imaginile si fonturile încorporate, astfel încât pagina sa poata fi deschisa si singura.
|
||||
Largest inlined asset (bytes):
|
||||
Cea mai mare resursa încorporata (octeti):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
O resursa mai mare decât aceasta dimensiune pastreaza o legatura obisnuita; lasati gol pentru valoarea implicita de 10485760 de octeti.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Èìÿ WARC-àðõèâà:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Íåîáÿçàòåëüíîå áàçîâîå èìÿ WARC-àðõèâà; îñòàâüòå ïóñòûì äëÿ àâòîìàòè÷åñêîãî èìåíîâàíèÿ â âûõîäíîì êàòàëîãå.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Âñòðàèâàòü ðåñóðñû êàê data: URI (ñàìîñòîÿòåëüíûå ñòðàíèöû)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ïîñëå çàâåðøåíèÿ çåðêàëèðîâàíèÿ êàæäàÿ ñîõðàí¸ííàÿ ñòðàíèöà ïåðåçàïèñûâàåòñÿ ñî âñòðîåííûìè òàáëèöàìè ñòèëåé, ñöåíàðèÿìè, èçîáðàæåíèÿìè è øðèôòàìè, ÷òîáû ñòðàíèöó ìîæíî áûëî îòêðûòü îòäåëüíî.
|
||||
Largest inlined asset (bytes):
|
||||
Íàèáîëüøèé âñòðàèâàåìûé ðåñóðñ (áàéòû):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Ðåñóðñ áîëüøå ýòîãî ðàçìåðà ñîõðàíÿåò îáû÷íóþ ññûëêó; îñòàâüòå ïóñòûì äëÿ çíà÷åíèÿ ïî óìîë÷àíèþ 10485760 áàéò.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Názov archívu WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Voliteµný základný názov archívu WARC; ponechajte prázdne pre automatické pomenovanie vo výstupnom adresári.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Vlo¾i» zdroje ako URI data: (samostatné stránky)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Po dokonèení zrkadlenia prepísa» ka¾dú ulo¾enú stránku s vlo¾enými ¹týlmi, skriptmi, obrázkami a písmami, aby sa stránka dala otvori» aj samostatne.
|
||||
Largest inlined asset (bytes):
|
||||
Najväè¹í vlo¾ený zdroj (bajty):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Zdroj väè¹í ne¾ táto veµkos» si ponechá be¾ný odkaz; ponechajte prázdne pre predvolených 10485760 bajtov.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
Ime arhiva WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Neobvezno osnovno ime arhiva WARC; pustite prazno za samodejno poimenovanje v izhodni mapi.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Vgradi vire kot data: URI (samostojne strani)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Ko je zrcaljenje koncano, se vsaka shranjena stran prepise z vgrajenimi slogovnimi listi, skripti, slikami in pisavami, tako da jo je mogoce odpreti tudi samostojno.
|
||||
Largest inlined asset (bytes):
|
||||
Najvecji vgrajeni vir (bajti):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Vir, vecji od te velikosti, ohrani obicajno povezavo; pustite prazno za privzetih 10485760 bajtov.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC-arkivets namn:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valfritt basnamn för WARC-arkivet; lämna tomt för att namnge det automatiskt i utdatakatalogen.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Bädda in resurser som data:-URI:er (fristående sidor)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
När speglingen är klar skrivs varje sparad sida om med sina formatmallar, skript, bilder och teckensnitt inbäddade, så att en sida också kan öppnas för sig.
|
||||
Largest inlined asset (bytes):
|
||||
Största inbäddade resurs (byte):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
En resurs över den här storleken behåller en vanlig länk; lämna tomt för standardvärdet 10485760 byte.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC arþivi adý:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC arþivi için isteðe baðlý temel ad; çýktý dizininde otomatik adlandýrma için boþ býrakýn.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Kaynaklarý data: URI olarak göm (kendi kendine yeten sayfalar)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Yansýlama bittiðinde, kaydedilen her sayfa stil sayfalarý, betikleri, görselleri ve yazý tipleri gömülü olarak yeniden yazýlýr; böylece sayfa tek baþýna da açýlabilir.
|
||||
Largest inlined asset (bytes):
|
||||
En büyük gömülü kaynak (bayt):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Bu boyutun üzerindeki bir kaynak sýradan baðlantýsýný korur; 10485760 baytlýk varsayýlan için boþ býrakýn.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
²ì'ÿ WARC-àðõ³âó:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Íåîáîâ'ÿçêîâà áàçîâà íàçâà WARC-àðõ³âó; çàëèøòå ïîðîæí³ì äëÿ àâòîìàòè÷íîãî íàéìåíóâàííÿ ó âèõ³äíîìó êàòàëîç³.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Âáóäîâóâàòè ðåñóðñè ÿê data: URI (ñàìîñò³éí³ ñòîð³íêè)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
ϳñëÿ çàâåðøåííÿ äçåðêàëþâàííÿ êîæíà çáåðåæåíà ñòîð³íêà ïåðåçàïèñóºòüñÿ ç âáóäîâàíèìè òàáëèöÿìè ñòèë³â, ñöåíàð³ÿìè, çîáðàæåííÿìè òà øðèôòàìè, ùîá ñòîð³íêó ìîæíà áóëî â³äêðèòè îêðåìî.
|
||||
Largest inlined asset (bytes):
|
||||
Íàéá³ëüøèé âáóäîâàíèé ðåñóðñ (áàéòè):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Ðåñóðñ, á³ëüøèé çà öåé ðîçì³ð, çáåð³ãຠçâè÷àéíå ïîñèëàííÿ; çàëèøòå ïîðîæí³ì äëÿ òèïîâîãî çíà÷åííÿ 10485760 áàéò³â.
|
||||
|
||||
@@ -964,3 +964,11 @@ WARC archive name:
|
||||
WARC arxivi nomi:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC arxivi uchun ixtiyoriy asosiy nom; chiqish katalogida avtomatik nomlash uchun bo'sh qoldiring.
|
||||
Inline assets as data: URIs (self-contained pages)
|
||||
Resurslarni data: URI sifatida joylash (mustaqil sahifalar)
|
||||
Once the mirror is finished, rewrite every saved page with its stylesheets, scripts, images and fonts embedded, so a page can also be opened on its own.
|
||||
Nusxalash tugagach, har bir saqlangan sahifa uslublar jadvallari, skriptlar, rasmlar va shriftlar joylangan holda qayta yoziladi, shunda sahifani alohida ham ochish mumkin.
|
||||
Largest inlined asset (bytes):
|
||||
Eng katta joylangan resurs (bayt):
|
||||
An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
|
||||
Bu o’lchamdan katta resurs oddiy havolani saqlab qoladi; standart 10485760 bayt uchun bo’sh qoldiring.
|
||||
|
||||
@@ -71,7 +71,9 @@ static int mysavename(t_hts_callbackarg * carg, httrackp * opt,
|
||||
for(j = 0; iisBogus[i][j] == a[j] && iisBogus[i][j] != '\0'; j++) ;
|
||||
if (iisBogus[i][j] == '\0'
|
||||
&& (a[j] == '\0' || a[j] == '/' || a[j] == '\\')) {
|
||||
strncpy(a, iisBogusReplace[i], strlen(iisBogusReplace[i]));
|
||||
/* j bytes matched, so j fit: copying j cannot overrun whatever the
|
||||
table holds, and the tail must survive untouched */
|
||||
memcpy(a, iisBogusReplace[i], (size_t) j);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "23 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "26 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -40,6 +40,7 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-%Z, \-\-single\-file\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
[ \fB\-KN, \-\-keep\-links[=N]\fR ]
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
@@ -198,6 +199,8 @@ delayed type check, don't make any link test but wait for files download to star
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%Z
|
||||
after the mirror, rewrite each saved page with its stylesheets, scripts, images and fonts inlined as data: URIs, so any page opens by double\-click anywhere (links between pages stay relative; audio and video stay links); \-\-single\-file\-max\-size N caps each asset (default 10485760 bytes). %M is the better container where a Chromium\-family browser is a given: one archive, no base64 tax on text, a shared asset stored once (\-\-single\-file)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
|
||||
@@ -32,7 +32,9 @@ AM_LDFLAGS = \
|
||||
|
||||
bin_PROGRAMS = proxytrack httrack htsserver
|
||||
|
||||
httrack_LDADD = $(THREADS_LIBS) libhttrack.la
|
||||
httrack_SOURCES = httrack.c htsbacktrace.c htsbacktrace.h
|
||||
# $(DL_LIBS): dladdr() in the crash handler, still in libdl on pre-2.34 glibc.
|
||||
httrack_LDADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
|
||||
htsserver_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) libhttrack.la
|
||||
proxytrack_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS)
|
||||
|
||||
@@ -46,7 +48,8 @@ htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
|
||||
|
||||
lib_LTLIBRARIES = libhttrack.la
|
||||
|
||||
htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h \
|
||||
htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h htsstats.h \
|
||||
htscmdline.c htscmdline.h \
|
||||
htsurlport.c htsurlport.h
|
||||
proxytrack_SOURCES = proxy/main.c \
|
||||
proxy/proxytrack.c proxy/store.c \
|
||||
@@ -60,21 +63,21 @@ whttrackrun_SCRIPTS = webhttrack
|
||||
libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htscache_selftest.c htsdns_selftest.c htsselftest.c \
|
||||
htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
|
||||
htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htswarc.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmd5.c htscodec.c htswarc.c htssinglefile.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
|
||||
htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h \
|
||||
htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
|
||||
htscmdline.h htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
|
||||
htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
|
||||
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htsproxy.h htszlib.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htssinglefile.h htsproxy.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
htscharset.h punycode.h htsencoding.h \
|
||||
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
|
||||
@@ -84,7 +87,7 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(BROTLI_LIBS) $(ZSTD_LIBS)
|
||||
libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
|
||||
libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
|
||||
|
||||
EXTRA_DIST = httrack.h webhttrack \
|
||||
EXTRA_DIST = httrack.h htsstats.h webhttrack \
|
||||
version.rc \
|
||||
libhttrack.rc \
|
||||
httrack.rc \
|
||||
|
||||
@@ -123,6 +123,10 @@ const char *hts_optalias[][4] = {
|
||||
{"warc-cdxj", "-%rc", "single", ""},
|
||||
{"wacz", "-%rz", "single",
|
||||
"package the WARC archive, CDXJ index and pages as a WACZ file"},
|
||||
{"single-file", "-%Z", "single",
|
||||
"after the mirror, inline each page's assets as data: URIs"},
|
||||
{"single-file-max-size", "-%Zs", "param1",
|
||||
"per-asset cap for --single-file, in bytes (implies it; default 10485760)"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
|
||||
105
src/htsback.c
105
src/htsback.c
@@ -125,6 +125,22 @@ void back_free(struct_back ** sback) {
|
||||
above a normal handshake. The last candidate still gets the full timeout. */
|
||||
#define HTS_CONNECT_FALLBACK_TIMEOUT 10
|
||||
|
||||
void back_read_ftp_result(FILE *fp, htsblk *r) {
|
||||
size_t j = 0;
|
||||
|
||||
if (fscanf(fp, "%d ", &r->statuscode) != 1)
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
// an external helper writes this file: stop at capacity, not at EOF
|
||||
while (j + 1 < sizeof(r->msg)) {
|
||||
const int c = fgetc(fp);
|
||||
|
||||
if (c == EOF)
|
||||
break;
|
||||
r->msg[j++] = (char) c;
|
||||
}
|
||||
r->msg[j] = '\0';
|
||||
}
|
||||
|
||||
int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
|
||||
int timeout) {
|
||||
int deadline;
|
||||
@@ -541,9 +557,15 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
|
||||
const char *ext) {
|
||||
// do not use tempnam() but a regular filename
|
||||
back->tmpfile_buffer[0] = '\0';
|
||||
if (back->url_sav != NULL && back->url_sav[0] != '\0') {
|
||||
snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
|
||||
back->url_sav, ext);
|
||||
if (back->url_sav[0] != '\0') {
|
||||
/* same capacity as url_sav, so truncation drops the extension and aliases
|
||||
the temp name onto the live file that back_finalize_backup() UNLINKs */
|
||||
if (!sprintfbuff(back->tmpfile_buffer, "%s.%s", back->url_sav, ext)) {
|
||||
hts_log_print(opt, LOG_WARNING, "temporary filename too long for %s",
|
||||
back->url_sav);
|
||||
back->tmpfile_buffer[0] = '\0';
|
||||
return -1;
|
||||
}
|
||||
back->tmpfile = back->tmpfile_buffer;
|
||||
if (structcheck(back->tmpfile) != 0) {
|
||||
hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
|
||||
@@ -551,8 +573,15 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
|
||||
StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
|
||||
/* truncation here would collide distinct tmpnameid's onto one name */
|
||||
if (!sprintfbuff(back->tmpfile_buffer, "%s/tmp%d.%s",
|
||||
StringBuff(opt->path_html_utf8), opt->state.tmpnameid++,
|
||||
ext)) {
|
||||
hts_log_print(opt, LOG_WARNING, "temporary filename too long in %s",
|
||||
StringBuff(opt->path_html_utf8));
|
||||
back->tmpfile_buffer[0] = '\0';
|
||||
return -1;
|
||||
}
|
||||
back->tmpfile = back->tmpfile_buffer;
|
||||
}
|
||||
/* OK */
|
||||
@@ -887,8 +916,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
HTS_STAT.stat_bytes += back[p].r.size;
|
||||
HTS_STAT.stat_files++;
|
||||
hts_log_print(opt, LOG_TRACE, "added file %s%s => %s",
|
||||
back[p].url_adr, back[p].url_fil,
|
||||
back[p].url_sav != NULL ? back[p].url_sav : "");
|
||||
back[p].url_adr, back[p].url_fil, back[p].url_sav);
|
||||
}
|
||||
if ((!back[p].r.notmodified) && (opt->is_update)) {
|
||||
HTS_STAT.stat_updated_files++; // page modifiée
|
||||
@@ -2302,26 +2330,24 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
|
||||
&& slot_can_be_finalized(opt, &back[i]);
|
||||
int may_serialize = slot_can_be_cached_on_disk(&back[i]);
|
||||
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"back[%03d]: may_clean=%d, may_finalize_disk=%d, may_serialize=%d:"
|
||||
LF "\t"
|
||||
"finalized(%d), status(%d), locked(%d), delayed(%d), test(%d), "
|
||||
LF "\t"
|
||||
"statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), "
|
||||
LF "\t" "contenttype(%s), url(%s%s), save(%s)", i,
|
||||
may_clean, may_finalize, may_serialize,
|
||||
back[i].finalized, back[i].status, back[i].locked,
|
||||
IS_DELAYED_EXT(back[i].url_sav), back[i].testmode,
|
||||
back[i].r.statuscode, (int) back[i].r.size,
|
||||
back[i].r.is_write, may_be_hypertext_mime(opt,
|
||||
back[i].r.
|
||||
contenttype,
|
||||
back[i].
|
||||
url_fil),
|
||||
/* */
|
||||
back[i].r.contenttype, back[i].url_adr,
|
||||
back[i].url_fil,
|
||||
back[i].url_sav ? back[i].url_sav : "<null>");
|
||||
hts_log_print(
|
||||
opt, LOG_DEBUG,
|
||||
"back[%03d]: may_clean=%d, may_finalize_disk=%d, "
|
||||
"may_serialize=%d:" LF "\t"
|
||||
"finalized(%d), status(%d), locked(%d), delayed(%d), "
|
||||
"test(%d), " LF "\t"
|
||||
"statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), " LF
|
||||
"\t"
|
||||
"contenttype(%s), url(%s%s), save(%s)",
|
||||
i, may_clean, may_finalize, may_serialize, back[i].finalized,
|
||||
back[i].status, back[i].locked, IS_DELAYED_EXT(back[i].url_sav),
|
||||
back[i].testmode, back[i].r.statuscode, (int) back[i].r.size,
|
||||
back[i].r.is_write,
|
||||
may_be_hypertext_mime(opt, back[i].r.contenttype,
|
||||
back[i].url_fil),
|
||||
/* */
|
||||
back[i].r.contenttype, back[i].url_adr, back[i].url_fil,
|
||||
back[i].url_sav);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2857,7 +2883,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// new session
|
||||
back[i].r.ssl_con = SSL_new(openssl_ctx);
|
||||
if (back[i].r.ssl_con) {
|
||||
const char* hostname = jump_protocol_const(back[i].url_adr);
|
||||
/* non-const twin: the OpenSSL macro casts the qualifier away */
|
||||
char *hostname = jump_protocol(back[i].url_adr);
|
||||
// some servers expect the hostname on the clienthello (SNI TLS extension)
|
||||
SSL_set_tlsext_host_name(back[i].r.ssl_con, hostname);
|
||||
SSL_clear(back[i].r.ssl_con);
|
||||
@@ -2949,7 +2976,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].r.msg[0] = '\0';
|
||||
strncatbuff(back[i].r.msg, tmp, sizeof(back[i].r.msg) - 2);
|
||||
if (!strnotempty(back[i].r.msg)) {
|
||||
sprintf(back[i].r.msg, "SSL/TLS error %d", err_code);
|
||||
htsblk_failf(&back[i].r, "SSL/TLS error %d", err_code);
|
||||
}
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
@@ -3022,16 +3049,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
FOPEN(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer, ".ok"),
|
||||
"rb");
|
||||
if (fp) {
|
||||
int j = 0;
|
||||
|
||||
fscanf(fp, "%d ", &(back[i].r.statuscode));
|
||||
while(!feof(fp)) {
|
||||
int c = fgetc(fp);
|
||||
|
||||
if (c != EOF)
|
||||
back[i].r.msg[j++] = c;
|
||||
}
|
||||
back[i].r.msg[j++] = '\0';
|
||||
back_read_ftp_result(fp, &back[i].r);
|
||||
fclose(fp);
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer, ".ok"));
|
||||
strcpybuff(fconcat
|
||||
@@ -3332,10 +3350,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
deleteaddr(&back[i].r);
|
||||
if (back[i].r.size < back[i].r.totalsize)
|
||||
back[i].r.statuscode = STATUSCODE_CONNERROR; // recatch
|
||||
sprintf(back[i].r.msg,
|
||||
"Incorrect length (" LLintP " Bytes, " LLintP
|
||||
" expected)", (LLint) back[i].r.size,
|
||||
(LLint) back[i].r.totalsize);
|
||||
htsblk_failf(&back[i].r,
|
||||
"Incorrect length (" LLintP " Bytes, " LLintP
|
||||
" expected)",
|
||||
(LLint) back[i].r.size,
|
||||
(LLint) back[i].r.totalsize);
|
||||
} else {
|
||||
// Un warning suffira..
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
|
||||
@@ -74,6 +74,10 @@ void back_free(struct_back ** sback);
|
||||
// backing
|
||||
#define BACK_ADD_TEST "(dummy)"
|
||||
#define BACK_ADD_TEST2 "(dummy2)"
|
||||
/* Parse an external FTP helper's "<statuscode> <message>" result file into r,
|
||||
clipping the message to r->msg. */
|
||||
void back_read_ftp_result(FILE *fp, htsblk *r);
|
||||
|
||||
int back_index(httrackp * opt, struct_back * sback, const char *adr, const char *fil,
|
||||
const char *sav);
|
||||
int back_available(const struct_back * sback);
|
||||
|
||||
238
src/htsbacktrace.c
Normal file
238
src/htsbacktrace.c
Normal file
@@ -0,0 +1,238 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: crash backtrace printer */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
/* Before every header: glibc gates dladdr() on it. */
|
||||
#if defined(__linux) && !defined(_GNU_SOURCE)
|
||||
#define _GNU_SOURCE
|
||||
#endif
|
||||
|
||||
#include "htsbacktrace.h"
|
||||
|
||||
#include "htsglobal.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#ifdef _WIN32
|
||||
#include <io.h> /* write */
|
||||
#endif
|
||||
#ifdef HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#if (defined(__linux) && defined(HAVE_EXECINFO_H))
|
||||
#include <dlfcn.h>
|
||||
#include <errno.h>
|
||||
#include <execinfo.h>
|
||||
#include <signal.h>
|
||||
#include <time.h>
|
||||
#include <sys/wait.h>
|
||||
#define USES_BACKTRACE
|
||||
#endif
|
||||
|
||||
#ifdef USES_BACKTRACE
|
||||
#define BT_MAX_FRAMES 64 /* frames we try to name */
|
||||
#define BT_MAX_MODULES 8 /* distinct modules, one child each */
|
||||
#define BT_HEX_SIZE 19 /* "0x" + 16 nibbles + NUL */
|
||||
#define BT_PATH_SIZE 1024 /* module path; longer is skipped */
|
||||
#define BT_WAIT_TICKS 300 /* 10ms ticks, shared: cap a slow child */
|
||||
#define BT_NO_SYMBOLIZER 127 /* child exit: execvp() found none */
|
||||
|
||||
static hts_boolean symbolize_crash = HTS_TRUE;
|
||||
|
||||
/* "0x"-prefixed hex: the handler must stay stdio-free. */
|
||||
static void print_hex(char *buffer, uintptr_t value) {
|
||||
static const char digits[] = "0123456789abcdef";
|
||||
size_t i = 2, a, b;
|
||||
|
||||
buffer[0] = '0';
|
||||
buffer[1] = 'x';
|
||||
do {
|
||||
buffer[i++] = digits[value & 0xf];
|
||||
value >>= 4;
|
||||
} while (value != 0);
|
||||
buffer[i] = '\0';
|
||||
for (a = 2, b = i - 1; a < b; a++, b--) {
|
||||
const char c = buffer[a];
|
||||
|
||||
buffer[a] = buffer[b];
|
||||
buffer[b] = c;
|
||||
}
|
||||
}
|
||||
|
||||
/* HTS_FALSE if src does not fit: a truncated module path would point the
|
||||
symbolizer at the wrong file. */
|
||||
static hts_boolean copy_bounded(char *dest, size_t size, const char *src) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < size - 1 && src[i] != '\0'; i++) {
|
||||
dest[i] = src[i];
|
||||
}
|
||||
dest[i] = '\0';
|
||||
return src[i] == '\0' ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Run the symbolizer on argv, output on fd, within *budget ticks. HTS_FALSE
|
||||
only if none could be run at all; otherwise silent, the raw trace stands. */
|
||||
static hts_boolean spawn_symbolizer(char **argv, int fd, int *budget) {
|
||||
const pid_t pid = fork();
|
||||
int status = 0;
|
||||
|
||||
if (pid == -1)
|
||||
return HTS_FALSE;
|
||||
if (pid == 0) {
|
||||
static char llvm_prog[] = "llvm-symbolizer";
|
||||
static char llvm_opts[] = "-p";
|
||||
|
||||
dup2(fd, 1); /* both symbolizers write on stdout */
|
||||
execvp(argv[0], argv);
|
||||
argv[0] = llvm_prog; /* an LLVM-only install ships no addr2line */
|
||||
argv[1] = llvm_opts;
|
||||
execvp(argv[0], argv);
|
||||
_exit(BT_NO_SYMBOLIZER);
|
||||
}
|
||||
for (; *budget > 0; (*budget)--) {
|
||||
const struct timespec tick = {0, 10 * 1000 * 1000};
|
||||
const pid_t reaped = waitpid(pid, &status, WNOHANG);
|
||||
|
||||
if (reaped == pid)
|
||||
return WIFEXITED(status) && WEXITSTATUS(status) == BT_NO_SYMBOLIZER
|
||||
? HTS_FALSE
|
||||
: HTS_TRUE;
|
||||
if (reaped == -1 && errno != EINTR)
|
||||
return HTS_TRUE;
|
||||
nanosleep(&tick, NULL);
|
||||
}
|
||||
kill(pid, SIGKILL);
|
||||
waitpid(pid, NULL, 0);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Name the frames backtrace_symbols_fd() leaves as module+offset:
|
||||
-fvisibility=hidden keeps them out of .dynsym, but DWARF has them. dladdr()
|
||||
is not formally async-signal-safe; accepted, this path is already fatal. */
|
||||
static void symbolize_backtrace(void *const *stack, int size, int fd) {
|
||||
static char prog[] = "addr2line";
|
||||
static char opts[] = "-Cfipa";
|
||||
static char dashe[] = "-e";
|
||||
char hex[BT_MAX_FRAMES][BT_HEX_SIZE];
|
||||
const void *base[BT_MAX_FRAMES];
|
||||
const char *name[BT_MAX_FRAMES];
|
||||
hts_boolean grouped[BT_MAX_FRAMES];
|
||||
char module[BT_PATH_SIZE];
|
||||
char *argv[4 + BT_MAX_FRAMES + 1];
|
||||
int budget = BT_WAIT_TICKS;
|
||||
int i, spawned;
|
||||
|
||||
if (size > BT_MAX_FRAMES)
|
||||
size = BT_MAX_FRAMES;
|
||||
|
||||
for (i = 0; i < size; i++) {
|
||||
Dl_info info;
|
||||
|
||||
grouped[i] = HTS_TRUE; /* skipped unless dladdr() places the frame */
|
||||
if (dladdr(stack[i], &info) == 0 || info.dli_fname == NULL ||
|
||||
info.dli_fname[0] == '\0')
|
||||
continue;
|
||||
base[i] = info.dli_fbase;
|
||||
name[i] = info.dli_fname;
|
||||
print_hex(hex[i], (uintptr_t) ((const char *) stack[i] -
|
||||
(const char *) info.dli_fbase));
|
||||
grouped[i] = HTS_FALSE;
|
||||
}
|
||||
|
||||
/* One child per module: addr2line takes a single -e. Each frame is claimed
|
||||
once, so argc cannot exceed argv[]. */
|
||||
for (spawned = 0; spawned < BT_MAX_MODULES; spawned++) {
|
||||
int first, j, argc = 0;
|
||||
|
||||
for (first = 0; first < size && grouped[first]; first++)
|
||||
;
|
||||
if (first >= size)
|
||||
break;
|
||||
argv[argc++] = prog;
|
||||
argv[argc++] = opts;
|
||||
argv[argc++] = dashe;
|
||||
argv[argc++] = module;
|
||||
for (j = first; j < size; j++) {
|
||||
if (grouped[j] || base[j] != base[first])
|
||||
continue;
|
||||
grouped[j] = HTS_TRUE;
|
||||
argv[argc++] = hex[j];
|
||||
}
|
||||
argv[argc] = NULL;
|
||||
/* access(): skip pseudo-modules like linux-vdso, which have no file and
|
||||
would draw nothing but an addr2line complaint. */
|
||||
if (copy_bounded(module, sizeof(module), name[first]) &&
|
||||
access(module, R_OK) == 0) {
|
||||
const size_t len = strlen(module);
|
||||
|
||||
/* addr2line -a prints offsets only: say which module they are in. */
|
||||
(void) (write(fd, module, len) == (ssize_t) len);
|
||||
(void) (write(fd, ":\n", 2) == 2);
|
||||
if (!spawn_symbolizer(argv, fd, &budget))
|
||||
break; /* no symbolizer: stop at one header */
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void hts_backtrace_init(void) {
|
||||
#ifdef USES_BACKTRACE
|
||||
symbolize_crash =
|
||||
getenv("HTTRACK_NO_SYMBOLIZE") == NULL ? HTS_TRUE : HTS_FALSE;
|
||||
#endif
|
||||
}
|
||||
|
||||
void hts_print_backtrace(int fd) {
|
||||
#ifdef USES_BACKTRACE
|
||||
void *stack[256];
|
||||
const int size = backtrace(stack, sizeof(stack) / sizeof(stack[0]));
|
||||
|
||||
/* A fault inside the handler lands back here: symbolizing twice interleaves
|
||||
two traces on fd and spends a second budget. */
|
||||
static volatile sig_atomic_t entered = 0;
|
||||
|
||||
if (size != 0) {
|
||||
backtrace_symbols_fd(stack, size, fd);
|
||||
if (symbolize_crash && entered == 0) {
|
||||
entered = 1;
|
||||
symbolize_backtrace(stack, size, fd);
|
||||
entered = 0;
|
||||
}
|
||||
}
|
||||
#else
|
||||
const char msg[] = "No stack trace available on this OS :(\n";
|
||||
|
||||
if (write(fd, msg, sizeof(msg) - 1) != sizeof(msg) - 1) {
|
||||
/* sorry GCC */
|
||||
}
|
||||
#endif
|
||||
}
|
||||
45
src/htsbacktrace.h
Normal file
45
src/htsbacktrace.h
Normal file
@@ -0,0 +1,45 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: crash backtrace printer */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSBACKTRACE_DEFH
|
||||
#define HTSBACKTRACE_DEFH
|
||||
|
||||
/* Sample HTTRACK_NO_SYMBOLIZE before any crash: getenv() is not signal-safe.
|
||||
Call once, from the process that installs the fatal-signal handlers. */
|
||||
void hts_backtrace_init(void);
|
||||
|
||||
/* Write the calling thread's stack to fd, callable from a fatal signal handler:
|
||||
raw frames first, then whatever an external symbolizer can name. Allocates
|
||||
nothing; prints a one-line notice where the OS has no backtrace(). */
|
||||
void hts_print_backtrace(int fd);
|
||||
|
||||
#endif
|
||||
@@ -142,10 +142,12 @@ struct cache_back_zip_entry {
|
||||
int compressionMethod;
|
||||
};
|
||||
|
||||
/* A corrupt cache can carry a field wider than ours; clipping it keeps the
|
||||
entry, where aborting would take the crawl down. */
|
||||
#define ZIP_READFIELD_STRING(line, value, refline, refvalue, refvalue_size) \
|
||||
do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
strlcpybuff(refvalue, value, refvalue_size); \
|
||||
(void) strclipbuff(refvalue, refvalue_size, value); \
|
||||
line[0] = '\0'; \
|
||||
} \
|
||||
} while (0)
|
||||
@@ -629,8 +631,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
// File exists on disk with declared cache name (this is expected!)
|
||||
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) { // un fichier existe déja
|
||||
// Expected size ?
|
||||
const size_t fsize =
|
||||
fsize_utf8(fconv(catbuff, sizeof(catbuff), previous_save));
|
||||
const LLint fsize = fsize_utf8(
|
||||
fconv(catbuff, sizeof(catbuff), previous_save));
|
||||
if (fsize == r.size) {
|
||||
// Target name is the previous name, and the file looks good: nothing to do!
|
||||
if (strcmp(previous_save, target_save) == 0) {
|
||||
@@ -666,7 +668,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
// Suppose a broken mirror, with a file being renamed: OK
|
||||
else if (fexist_utf8(fconv(catbuff, sizeof(catbuff), target_save))) {
|
||||
// Expected size ?
|
||||
const size_t fsize = fsize_utf8(fconv(catbuff, sizeof(catbuff), target_save));
|
||||
const LLint fsize =
|
||||
fsize_utf8(fconv(catbuff, sizeof(catbuff), target_save));
|
||||
|
||||
if (fsize == r.size) {
|
||||
// So far so good
|
||||
@@ -1294,12 +1297,17 @@ char *readfile2(const char *fil, LLint * size) {
|
||||
}
|
||||
|
||||
/* Note: utf-8 */
|
||||
char *readfile_utf8(const char *fil) {
|
||||
char *readfile_utf8(const char *fil) { return readfile2_utf8(fil, NULL); }
|
||||
|
||||
/* Note: utf-8 */
|
||||
char *readfile2_utf8(const char *fil, LLint *size) {
|
||||
char *adr = NULL;
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const LLint len = fsize_utf8(fil);
|
||||
const size_t buflen = len >= 0 ? llint_to_size_t(len) : (size_t) -1;
|
||||
|
||||
if (size != NULL)
|
||||
*size = len;
|
||||
if (buflen != (size_t) -1) { // exists, and is addressable (see readfile2)
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), fil), "rb");
|
||||
|
||||
@@ -1440,7 +1448,7 @@ int cache_brstr(char *adr, char *s, size_t s_size) {
|
||||
|
||||
/* binput bounded to a NUL-terminated buffer: refuse to start a read at or
|
||||
past `end`, so a prior over-advance can't walk a cache-index parse OOB. */
|
||||
int cache_binput(char *adr, const char *end, char *s, int max) {
|
||||
int cache_binput(const char *adr, const char *end, char *s, int max) {
|
||||
if (adr >= end) {
|
||||
s[0] = '\0';
|
||||
return 0;
|
||||
|
||||
@@ -93,7 +93,7 @@ void cache_rstr(FILE *fp, char *s, size_t s_size);
|
||||
char *cache_rstr_addr(FILE * fp);
|
||||
int cache_brstr(char *adr, char *s, size_t s_size);
|
||||
/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
|
||||
int cache_binput(char *adr, const char *end, char *s, int max);
|
||||
int cache_binput(const char *adr, const char *end, char *s, int max);
|
||||
int cache_brint(char *adr, int *i);
|
||||
void cache_rint(FILE * fp, int *i);
|
||||
void cache_rLLint(FILE * fp, LLint * i);
|
||||
|
||||
@@ -1195,6 +1195,12 @@ int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
|
||||
|
||||
/* --- read-side corruption injection --------------------------------------- */
|
||||
|
||||
/* 100 'A's: a placeholder header line long enough to be overwritten by a
|
||||
forged, over-long X-StatusMessage of the same byte length. */
|
||||
#define CORRUPT_LONG_ETAG \
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" \
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
|
||||
/* canary read back intact after each corruption; victim gets the byte surgery
|
||||
*/
|
||||
#define CORRUPT_ADR "corrupt.example.com"
|
||||
@@ -1219,6 +1225,24 @@ static void corrupt_build(httrackp *opt) {
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build, but the victim carries a 100-char Etag placeholder. */
|
||||
static void corrupt_build_longetag(httrackp *opt) {
|
||||
cache_back cache;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
|
||||
"OK", "text/html", "utf-8", "", CORRUPT_LONG_ETAG, "", "",
|
||||
corrupt_body_b, strlen(corrupt_body_b));
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
|
||||
is later overwritten with a forged oversized X-Size (same byte length). */
|
||||
static void corrupt_build_etag(httrackp *opt) {
|
||||
@@ -1403,6 +1427,48 @@ static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
|
||||
return fail;
|
||||
}
|
||||
|
||||
/* An over-long field from a foreign cache must clip, not abort: the entry
|
||||
still reads, clipped to capacity, and the canary survives. */
|
||||
static int corrupt_expect_victim_clipped(httrackp *opt, size_t wantmsg,
|
||||
size_t wantlastmod, const char *what) {
|
||||
cache_back cache;
|
||||
htsblk v, c;
|
||||
char BIGSTK lv[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK lc[HTS_URLMAXSIZE * 2];
|
||||
int fail = 0;
|
||||
|
||||
selftest_open_for_read(&cache, opt);
|
||||
lv[0] = lc[0] = '\0';
|
||||
v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.html", "", lv, NULL, 1);
|
||||
if (v.statuscode != 200) {
|
||||
fprintf(stderr, "%s: %s: status %d, expected 200\n", selftest_tag, what,
|
||||
v.statuscode);
|
||||
fail++;
|
||||
}
|
||||
if (wantmsg != (size_t) -1 && strlen(v.msg) != wantmsg) {
|
||||
fprintf(stderr, "%s: %s: msg len %u, expected %u\n", selftest_tag, what,
|
||||
(unsigned) strlen(v.msg), (unsigned) wantmsg);
|
||||
fail++;
|
||||
}
|
||||
if (wantlastmod != (size_t) -1 && strlen(v.lastmodified) != wantlastmod) {
|
||||
fprintf(stderr, "%s: %s: lastmodified len %u, expected %u\n", selftest_tag,
|
||||
what, (unsigned) strlen(v.lastmodified), (unsigned) wantlastmod);
|
||||
fail++;
|
||||
}
|
||||
c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
|
||||
if (c.statuscode != 200) {
|
||||
fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
|
||||
c.statuscode);
|
||||
fail++;
|
||||
}
|
||||
if (v.adr != NULL)
|
||||
freet(v.adr);
|
||||
if (c.adr != NULL)
|
||||
freet(c.adr);
|
||||
selftest_close(&cache);
|
||||
return fail;
|
||||
}
|
||||
|
||||
/* One zip corruption case: build, patch, then check victim+canary in-session.
|
||||
*/
|
||||
static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
|
||||
@@ -1439,6 +1505,31 @@ int cache_corruption_selftest(httrackp *opt, const char *dir) {
|
||||
failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
|
||||
"garbled deflate stream");
|
||||
|
||||
/* A corrupt cache can hold a field wider than ours. Clipping keeps the
|
||||
entry; aborting would take the crawl down. Overwrite the placeholder Etag
|
||||
line in place, same byte length, so the zip offsets stay intact. */
|
||||
corrupt_build_longetag(opt);
|
||||
corrupt_patch(opt, "Etag: " CORRUPT_LONG_ETAG, 106,
|
||||
"X-StatusMessage: " /* 17 + 89 = 106 */
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
1, 1);
|
||||
failures +=
|
||||
corrupt_expect_victim_clipped(opt, sizeof(((htsblk *) 0)->msg) - 1,
|
||||
(size_t) -1, "over-long X-StatusMessage");
|
||||
|
||||
/* lastmodified[64] is narrower than msg[80]: one hardcoded clip length
|
||||
cannot satisfy both. */
|
||||
corrupt_build_longetag(opt);
|
||||
corrupt_patch(opt, "Etag: " CORRUPT_LONG_ETAG, 106,
|
||||
"Last-Modified: " /* 15 + 91 = 106 */
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
1, 1);
|
||||
failures += corrupt_expect_victim_clipped(
|
||||
opt, (size_t) -1, sizeof(((htsblk *) 0)->lastmodified) - 1,
|
||||
"over-long Last-Modified");
|
||||
|
||||
/* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
|
||||
truncates negative in the (int) cast the malloc uses: a wraparound alloc.
|
||||
cache_add asserts size fits an int, so such a value only reaches the reader
|
||||
|
||||
95
src/htscmdline.c
Normal file
95
src/htscmdline.c
Normal file
@@ -0,0 +1,95 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: command line splitter, shared by the engine and */
|
||||
/* htsserver */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#include "htscmdline.h"
|
||||
|
||||
#include "htssafe.h"
|
||||
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
|
||||
char **hts_split_cmdline(char *cmd, int *nargs) {
|
||||
size_t nsep = 0;
|
||||
size_t capacity;
|
||||
size_t r;
|
||||
size_t w;
|
||||
int argc = 0;
|
||||
hts_boolean quoted = HTS_FALSE;
|
||||
char **argv;
|
||||
|
||||
*nargs = 0;
|
||||
|
||||
/* fold the other separators, so counting them sizes the vector exactly */
|
||||
for (r = 0; cmd[r] != '\0'; r++) {
|
||||
if (cmd[r] == '\t' || cmd[r] == '\r' || cmd[r] == '\n') {
|
||||
cmd[r] = ' ';
|
||||
}
|
||||
if (cmd[r] == ' ') {
|
||||
nsep++;
|
||||
}
|
||||
}
|
||||
|
||||
/* at most one argument per separator, plus the leading one and the NULL */
|
||||
if (nsep > (size_t) INT_MAX - 1 || nsep > SIZE_MAX / sizeof(char *) - 2) {
|
||||
return NULL;
|
||||
}
|
||||
capacity = nsep + 2;
|
||||
argv = (char **) malloct(capacity * sizeof(char *));
|
||||
if (argv == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
argv[argc++] = cmd;
|
||||
for (r = 0, w = 0; cmd[r] != '\0';) {
|
||||
if (quoted && cmd[r] == '\\' &&
|
||||
(cmd[r + 1] == '\\' || cmd[r + 1] == '\"')) {
|
||||
r++;
|
||||
cmd[w++] = cmd[r++];
|
||||
} else if (cmd[r] == '\"') {
|
||||
quoted = !quoted;
|
||||
cmd[w++] = cmd[r++];
|
||||
} else if (cmd[r] == ' ' && !quoted) {
|
||||
cmd[w++] = '\0';
|
||||
assertf((size_t) argc < capacity - 1); /* the last slot holds the NULL */
|
||||
argv[argc++] = cmd + w;
|
||||
r++;
|
||||
} else {
|
||||
cmd[w++] = cmd[r++];
|
||||
}
|
||||
}
|
||||
cmd[w] = '\0';
|
||||
argv[argc] = NULL; /* callers may rely on argv[argc] == NULL */
|
||||
|
||||
*nargs = argc;
|
||||
return argv;
|
||||
}
|
||||
45
src/htscmdline.h
Normal file
45
src/htscmdline.h
Normal file
@@ -0,0 +1,45 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: command line splitter, shared by the engine and */
|
||||
/* htsserver */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSCMDLINE_DEFH
|
||||
#define HTSCMDLINE_DEFH
|
||||
|
||||
#include "htsglobal.h"
|
||||
|
||||
/* Split "cmd" in place into a NULL-terminated argv vector of *nargs entries,
|
||||
argv[0] being the program name and quotes left for the engine to strip.
|
||||
Returns a malloct'ed vector of pointers into cmd (freet the vector, never its
|
||||
entries), or NULL when it cannot be sized or allocated. */
|
||||
char **hts_split_cmdline(char *cmd, int *nargs);
|
||||
|
||||
#endif
|
||||
@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
|
||||
/* File defs */
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -447,13 +448,22 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
|
||||
const char *fil) {
|
||||
if (!*makeindex_done) {
|
||||
if (*makeindex_fp) {
|
||||
char BIGSTK tempo[1024];
|
||||
/* sized off link_escaped below: at the old flat 1024 a long first link
|
||||
produced a redirect to a clipped URL */
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2 + 64];
|
||||
if (makeindex_links == 1) {
|
||||
char BIGSTK link_escaped[HTS_URLMAXSIZE * 2];
|
||||
escape_uri_utf(makeindex_firstlink, link_escaped, sizeof(link_escaped));
|
||||
snprintf(tempo, sizeof(tempo),
|
||||
"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">" CRLF,
|
||||
link_escaped);
|
||||
/* no redirect beats one pointing at a clipped URL */
|
||||
if (!sprintfbuff(
|
||||
tempo,
|
||||
"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">" CRLF,
|
||||
link_escaped)) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"index redirect omitted: first link too long (%s)",
|
||||
makeindex_firstlink);
|
||||
tempo[0] = '\0';
|
||||
}
|
||||
} else
|
||||
tempo[0] = '\0';
|
||||
hts_template_format(*makeindex_fp, template_footer,
|
||||
@@ -696,17 +706,19 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// copier adresse(s) dans liste des adresses
|
||||
{
|
||||
char *a = url1;
|
||||
int primary_len = 8192;
|
||||
|
||||
if (StringNotEmpty(opt->filelist)) {
|
||||
primary_len += max(0, fsize_utf8(StringBuff(opt->filelist)) * 2);
|
||||
}
|
||||
primary_len += (int) strlen(url1) * 2;
|
||||
const LLint list_sz = StringNotEmpty(opt->filelist)
|
||||
? fsize_utf8(StringBuff(opt->filelist))
|
||||
: 0;
|
||||
/* two bytes reserved per list byte; -1 makes an undoublable size refused */
|
||||
const LLint list_room =
|
||||
list_sz > 0 ? (list_sz <= INT64_MAX / 2 ? list_sz * 2 : -1) : 0;
|
||||
const size_t primary_len =
|
||||
llint_grow_size_t(8192 + strlen(url1) * 2, list_room, 0);
|
||||
|
||||
// création de la première page, qui contient les liens de base à scanner
|
||||
// c'est plus propre et plus logique que d'entrer à la main les liens dans la pile
|
||||
// on bénéficie ainsi des vérifications et des tests du robot pour les liens "primaires"
|
||||
primary = (char *) malloct(primary_len);
|
||||
primary = primary_len != (size_t) -1 ? (char *) malloct(primary_len) : NULL;
|
||||
if (!primary) {
|
||||
printf("PANIC! : Not enough memory [%d]\n", __LINE__);
|
||||
XH_extuninit;
|
||||
@@ -887,7 +899,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
|
||||
if (filelist_buff != NULL) {
|
||||
int filelist_ptr = 0;
|
||||
size_t filelist_ptr = 0;
|
||||
int n = 0;
|
||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
||||
|
||||
@@ -2171,6 +2183,10 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
// fin purge!
|
||||
|
||||
/* --single-file: inline each page's assets now the tree is final, after the
|
||||
purge has deleted whatever this run dropped. */
|
||||
singlefile_process_mirror(opt);
|
||||
|
||||
// Indexation
|
||||
if (opt->kindex)
|
||||
index_finish(StringBuff(opt->path_html), opt->kindex);
|
||||
@@ -3632,6 +3648,10 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
to->warc_cdx = from->warc_cdx;
|
||||
to->warc_wacz = from->warc_wacz;
|
||||
|
||||
to->single_file = from->single_file;
|
||||
if (from->single_file_max_size > 0)
|
||||
to->single_file_max_size = from->single_file_max_size;
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
to->pause_max_ms = from->pause_max_ms;
|
||||
|
||||
@@ -409,6 +409,8 @@ char *readfile2(const char *fil, LLint * size);
|
||||
|
||||
char *readfile_utf8(const char *fil);
|
||||
|
||||
char *readfile2_utf8(const char *fil, LLint *size);
|
||||
|
||||
char *readfile_or(const char *fil, const char *defaultdata);
|
||||
|
||||
/* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
|
||||
|
||||
@@ -145,7 +145,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
int argv_url = -1; // ==0 : utiliser cache et doit.log
|
||||
char *argv_firsturl = NULL; // utilisé pour nommage par défaut
|
||||
char *url = NULL; // URLS séparées par un espace
|
||||
int url_sz = 65535;
|
||||
size_t url_sz = 65535;
|
||||
|
||||
// the parametres
|
||||
int httrack_logmode = 3; // ONE log file
|
||||
@@ -224,21 +224,22 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|
||||
/* create x_argvblk buffer for transformed command line */
|
||||
{
|
||||
int current_size = 0;
|
||||
int size;
|
||||
size_t current_size = 0;
|
||||
const LLint size = fsize("config");
|
||||
size_t blk_size;
|
||||
int na;
|
||||
|
||||
for(na = 0; na < argc; na++)
|
||||
current_size += (int) (strlen(argv[na]) + 1);
|
||||
if ((size = fsize("config")) > 0)
|
||||
current_size += size;
|
||||
x_argvblk = (char *) malloct(current_size + 32768);
|
||||
current_size += strlen(argv[na]) + 1;
|
||||
/* a huge file named "config" must saturate, not wrap, the capacity */
|
||||
blk_size = llint_grow_size_t(current_size, size > 0 ? size : 0, 32768);
|
||||
x_argvblk = blk_size != (size_t) -1 ? (char *) malloct(blk_size) : NULL;
|
||||
if (x_argvblk == NULL) {
|
||||
HTS_PANIC_PRINTF("Error, not enough memory");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
x_argvblk_size = (size_t) (current_size + 32768);
|
||||
x_argvblk_size = blk_size;
|
||||
x_argvblk[0] = '\0';
|
||||
x_ptr = 0;
|
||||
|
||||
@@ -1456,20 +1457,29 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
FILE *fp = FOPEN(argv[na], "rb");
|
||||
|
||||
if (fp != NULL) {
|
||||
int cl = (int) strlen(url);
|
||||
size_t cl = strlen(url);
|
||||
const size_t fzs = llint_to_size_t(fz);
|
||||
const size_t capa = llint_grow_size_t(cl, fz, 8192);
|
||||
|
||||
ensureUrlCapacity(url, url_sz, cl + fz + 8192);
|
||||
if (capa == (size_t) -1) {
|
||||
fclose(fp);
|
||||
HTS_PANIC_PRINTF("File url list too large");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
ensureUrlCapacity(url, url_sz, capa);
|
||||
if (cl > 0) { /* don't stick! (3.43) */
|
||||
url[cl] = ' ';
|
||||
cl++;
|
||||
}
|
||||
if (fread(url + cl, 1, fz, fp) != fz) {
|
||||
if (fread(url + cl, 1, fzs, fp) != fzs) {
|
||||
fclose(fp);
|
||||
HTS_PANIC_PRINTF("File url list could not be read");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
fclose(fp);
|
||||
*(url + cl + fz) = '\0';
|
||||
*(url + cl + fzs) = '\0';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1785,6 +1795,36 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
break;
|
||||
case 'Z': // single-file: inline each page's assets as data: URIs
|
||||
if (*(com + 1) == 's') { // --single-file-max-size N
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option single-file-max-size needs a blank "
|
||||
"space and a size");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
{ // reject non-numeric/negative/overflow; keep the default
|
||||
char *end;
|
||||
LLint v;
|
||||
|
||||
errno = 0;
|
||||
v = strtoll(argv[na], &end, 10);
|
||||
if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
|
||||
errno != ERANGE && v > 0)
|
||||
opt->single_file_max_size = v;
|
||||
}
|
||||
opt->single_file = HTS_TRUE;
|
||||
} else {
|
||||
opt->single_file = HTS_TRUE;
|
||||
if (*(com + 1) == '0') {
|
||||
opt->single_file = HTS_FALSE;
|
||||
com++;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
@@ -2296,8 +2336,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|
||||
} else { // URL/filters
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const int urlSize = (int) strlen(argv[na]);
|
||||
const int capa = (int) (strlen(url) + urlSize + 32);
|
||||
const size_t urlSize = strlen(argv[na]);
|
||||
const size_t capa = strlen(url) + urlSize + 32;
|
||||
|
||||
assertf(urlSize < HTS_URLMAXSIZE);
|
||||
if (urlSize < HTS_URLMAXSIZE) {
|
||||
|
||||
46
src/htsftp.c
46
src/htsftp.c
@@ -251,7 +251,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
// folding a nonsense port into 1..65535 fetches one the link never named;
|
||||
// an empty "host:" just means the default (#614)
|
||||
if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Invalid port: %s", a + 1);
|
||||
htsblk_failf(&back->r, "Invalid port: %s", a + 1);
|
||||
back->r.statuscode = STATUSCODE_INVALID; // permanent, unlike a DNS miss
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
@@ -262,8 +262,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
// récupérer adresse résolue
|
||||
strcpybuff(back->info, "host name");
|
||||
if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"Unable to get server's address: %s", error);
|
||||
htsblk_failf(&back->r, "Unable to get server's address: %s", error);
|
||||
back->r.statuscode = STATUSCODE_NON_FATAL;
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
@@ -332,18 +331,15 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "Bad password: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "Bad user name: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "Connection refused: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
|
||||
@@ -410,8 +406,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
// -- fin analyse de l'adresse IP et du port --
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "PASV incorrect: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
@@ -442,13 +437,11 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "EPSV incorrect: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "PASV/EPSV error: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
}
|
||||
@@ -554,8 +547,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
deletesoc(soc_dat);
|
||||
soc_dat = INVALID_SOCKET;
|
||||
//
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"RETR command error: %s", linejmp(line));
|
||||
htsblk_failf(&back->r, "RETR command error: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
@@ -573,13 +566,12 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"Unable to resolve IP %s: %s", adr_ip, error);
|
||||
htsblk_failf(&back->r, "Unable to resolve IP %s: %s", adr_ip,
|
||||
error);
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "PASV incorrect: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
#else
|
||||
@@ -603,13 +595,11 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"RETR command error: %s", linejmp(line));
|
||||
htsblk_failf(&back->r, "RETR command error: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "PORT command error: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
@@ -652,8 +642,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
len = 0; // fin
|
||||
break;
|
||||
case 0:
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
|
||||
timeout);
|
||||
htsblk_failf(&back->r, "Time out (%d)", timeout);
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
len = 0; // fin
|
||||
break;
|
||||
@@ -716,8 +705,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
strcpybuff(back->r.msg, "OK");
|
||||
back->r.statuscode = HTTP_OK;
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
|
||||
linejmp(line));
|
||||
htsblk_failf(&back->r, "RETR incorrect: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -72,7 +72,8 @@ Please visit our Website: http://www.httrack.com
|
||||
HTS_UNUSED: suppress unused-symbol warnings. HTS_STATIC: an unused-safe
|
||||
static. HTS_PRINTF_FUN(fmt, arg): mark a printf-like function so the
|
||||
compiler type-checks the format string at argument index fmt against the
|
||||
varargs starting at arg. */
|
||||
varargs starting at arg. HTS_CHECK_RESULT: the return value carries the only
|
||||
error signal, so dropping it is a bug; a (void) cast does not silence it. */
|
||||
#ifndef HTS_UNUSED
|
||||
#ifdef __GNUC__
|
||||
#define HTS_UNUSED __attribute__((unused))
|
||||
@@ -80,10 +81,13 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_STATIC static __attribute__((unused))
|
||||
|
||||
#define HTS_PRINTF_FUN(fmt, arg) __attribute__((format(printf, fmt, arg)))
|
||||
|
||||
#define HTS_CHECK_RESULT __attribute__((warn_unused_result))
|
||||
#else
|
||||
#define HTS_UNUSED
|
||||
#define HTS_STATIC static
|
||||
#define HTS_PRINTF_FUN(fmt, arg)
|
||||
#define HTS_CHECK_RESULT
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@@ -336,16 +340,20 @@ typedef int INTsys;
|
||||
#endif
|
||||
|
||||
/* Socket-handle type. An unsigned integer wide enough for a Windows SOCKET;
|
||||
a plain int file descriptor on POSIX. */
|
||||
a plain int file descriptor on POSIX. T_SOCP is its printf conversion,
|
||||
'%' included: unsigned __int64 on Win64 must not be printed with "%d". */
|
||||
#ifdef _WIN32
|
||||
#if defined(_WIN64)
|
||||
|
||||
typedef unsigned __int64 T_SOC;
|
||||
#define T_SOCP "%" PRIu64
|
||||
#else
|
||||
typedef unsigned __int32 T_SOC;
|
||||
#define T_SOCP "%" PRIu32
|
||||
#endif
|
||||
#else
|
||||
typedef int T_SOC;
|
||||
#define T_SOCP "%d"
|
||||
#endif
|
||||
|
||||
/* Buffer size for a printed network address (IPv4 or IPv6, NUL included). */
|
||||
|
||||
@@ -124,7 +124,9 @@ typedef struct help_wizard_buffers {
|
||||
char stropt[2048]; // options
|
||||
char stropt2[2048]; // options longues
|
||||
char strwild[2048]; // wildcards
|
||||
char cmd[4096];
|
||||
/* holds all four of the above plus separators: at 4096 a long answer set
|
||||
clipped the filters off the command line */
|
||||
char cmd[HTS_URLMAXSIZE * 2 + 3 * 2048 + 4];
|
||||
char str[256];
|
||||
char *argv[256];
|
||||
} help_wizard_buffers;
|
||||
@@ -156,9 +158,7 @@ void help_wizard(httrackp * opt) {
|
||||
char *a;
|
||||
|
||||
//
|
||||
if (urls == NULL || mainpath == NULL || projname == NULL || stropt == NULL
|
||||
|| stropt2 == NULL || strwild == NULL || cmd == NULL || str == NULL
|
||||
|| argv == NULL) {
|
||||
if (buffers == NULL) {
|
||||
fprintf(stderr, "* memory exhausted in %s, line %d\n", __FILE__, __LINE__);
|
||||
return;
|
||||
}
|
||||
@@ -251,6 +251,7 @@ void help_wizard(httrackp * opt) {
|
||||
strcatbuff(stropt2, "--update ");
|
||||
break;
|
||||
case 0:
|
||||
freet(buffers);
|
||||
return;
|
||||
break;
|
||||
}
|
||||
@@ -309,14 +310,23 @@ void help_wizard(httrackp * opt) {
|
||||
printf("\n");
|
||||
if (strlen(stropt) == 1)
|
||||
stropt[0] = '\0'; // aucune
|
||||
snprintf(cmd, sizeof(cmd), "%s %s %s %s", urls, stropt, stropt2, strwild);
|
||||
/* the tail is the filter list, and cmd is split into the argv handed to
|
||||
hts_main() below: a clipped line would silently widen the crawl */
|
||||
if (!sprintfbuff(cmd, "%s %s %s %s", urls, stropt, stropt2, strwild)) {
|
||||
printf("* command line too long (%d bytes max)\n",
|
||||
(int) sizeof(cmd) - 1);
|
||||
freet(buffers);
|
||||
return;
|
||||
}
|
||||
printf("---> Wizard command line: httrack %s\n\n", cmd);
|
||||
printf("Ready to launch the mirror? (Y/n) :");
|
||||
fflush(stdout);
|
||||
linput(stdin, str, 250);
|
||||
if (strnotempty(str)) {
|
||||
if (!((str[0] == 'y') || (str[0] == 'Y')))
|
||||
if (!((str[0] == 'y') || (str[0] == 'Y'))) {
|
||||
freet(buffers);
|
||||
return;
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
@@ -340,7 +350,7 @@ void help_wizard(httrackp * opt) {
|
||||
}
|
||||
|
||||
/* Free buffers */
|
||||
free(buffers);
|
||||
freet(buffers);
|
||||
#undef urls
|
||||
#undef mainpath
|
||||
#undef projname
|
||||
@@ -423,7 +433,8 @@ void help_catchurl(const char *dest_path) {
|
||||
}
|
||||
// former URL!
|
||||
{
|
||||
char BIGSTK finalurl[HTS_URLMAXSIZE * 2];
|
||||
/* url and dest are each HTS_URLMAXSIZE*2, plus the POSTTOK marker */
|
||||
char BIGSTK finalurl[HTS_URLMAXSIZE * 4 + 32];
|
||||
|
||||
inplace_escape_check_url(dest, sizeof(dest));
|
||||
snprintf(finalurl, sizeof(finalurl), "%s" POSTTOK "file:%s", url, dest);
|
||||
@@ -524,6 +535,13 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %Z after the mirror, rewrite each saved page with its "
|
||||
"stylesheets, scripts, images and fonts inlined as data: URIs, so "
|
||||
"any page opens by double-click anywhere (links between pages stay "
|
||||
"relative; audio and video stay links); --single-file-max-size N "
|
||||
"caps each asset (default 10485760 bytes). %M is the better "
|
||||
"container where a Chromium-family browser is a given: one archive, "
|
||||
"no base64 tax on text, a shared asset stored once");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
|
||||
58
src/htslib.c
58
src/htslib.c
@@ -37,6 +37,7 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htssinglefile.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -704,18 +705,16 @@ T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
|
||||
/* Check for errors */
|
||||
if (soc == INVALID_SOCKET) {
|
||||
if (retour) {
|
||||
if (retour->msg) {
|
||||
if (!strnotempty(retour->msg)) {
|
||||
if (!strnotempty(retour->msg)) {
|
||||
#ifdef _WIN32
|
||||
int last_errno = WSAGetLastError();
|
||||
int last_errno = WSAGetLastError();
|
||||
|
||||
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
|
||||
htsblk_failf(retour, "Connect error: %s", strerror(last_errno));
|
||||
#else
|
||||
int last_errno = errno;
|
||||
int last_errno = errno;
|
||||
|
||||
sprintf(retour->msg, "Connect error: %s", strerror(last_errno));
|
||||
htsblk_failf(retour, "Connect error: %s", strerror(last_errno));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2207,7 +2206,7 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
#if DEBUG
|
||||
printf("erreur gethostbyname\n");
|
||||
#endif
|
||||
if (retour && retour->msg) {
|
||||
if (retour != NULL) {
|
||||
#ifdef _WIN32
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to get server's address: %s", error);
|
||||
@@ -2238,17 +2237,17 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
DEBUG_W("socket()=%d\n" _(int) soc);
|
||||
#endif
|
||||
if (soc == INVALID_SOCKET) {
|
||||
if (retour && retour->msg) {
|
||||
if (retour != NULL) {
|
||||
#ifdef _WIN32
|
||||
int last_errno = WSAGetLastError();
|
||||
|
||||
sprintf(retour->msg, "Unable to create a socket: %s",
|
||||
strerror(last_errno));
|
||||
htsblk_failf(retour, "Unable to create a socket: %s",
|
||||
strerror(last_errno));
|
||||
#else
|
||||
int last_errno = errno;
|
||||
|
||||
sprintf(retour->msg, "Unable to create a socket: %s",
|
||||
strerror(last_errno));
|
||||
htsblk_failf(retour, "Unable to create a socket: %s",
|
||||
strerror(last_errno));
|
||||
#endif
|
||||
}
|
||||
return INVALID_SOCKET; // erreur création socket impossible
|
||||
@@ -2262,17 +2261,8 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
&bind_addr, &error) == NULL
|
||||
|| bind(soc, &SOCaddr_sockaddr(bind_addr),
|
||||
SOCaddr_size(bind_addr)) != 0) {
|
||||
if (retour && retour->msg) {
|
||||
#ifdef _WIN32
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to bind the specificied server address: %s",
|
||||
error);
|
||||
#else
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to bind the specificied server address: %s",
|
||||
error);
|
||||
#endif
|
||||
}
|
||||
snprintf(retour->msg, sizeof(retour->msg),
|
||||
"Unable to bind the specificied server address: %s", error);
|
||||
deletesoc(soc);
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
@@ -2319,17 +2309,17 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
#if HDEBUG
|
||||
printf("unable to connect!\n");
|
||||
#endif
|
||||
if (retour != NULL && retour->msg) {
|
||||
if (retour != NULL) {
|
||||
#ifdef _WIN32
|
||||
const int last_errno = WSAGetLastError();
|
||||
|
||||
sprintf(retour->msg, "Unable to connect to the server: %s",
|
||||
strerror(last_errno));
|
||||
htsblk_failf(retour, "Unable to connect to the server: %s",
|
||||
strerror(last_errno));
|
||||
#else
|
||||
const int last_errno = errno;
|
||||
|
||||
sprintf(retour->msg, "Unable to connect to the server: %s",
|
||||
strerror(last_errno));
|
||||
htsblk_failf(retour, "Unable to connect to the server: %s",
|
||||
strerror(last_errno));
|
||||
#endif
|
||||
}
|
||||
/* Close the socket and notify the error!!! */
|
||||
@@ -2598,13 +2588,15 @@ void deletesoc(T_SOC soc) {
|
||||
if (closesocket(soc) != 0) {
|
||||
int err = WSAGetLastError();
|
||||
|
||||
fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err));
|
||||
fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
|
||||
strerror(err));
|
||||
}
|
||||
#else
|
||||
if (close(soc) != 0) {
|
||||
const int err = errno;
|
||||
|
||||
fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err));
|
||||
fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
|
||||
strerror(err));
|
||||
}
|
||||
#endif
|
||||
#if HTS_WIDE_DEBUG
|
||||
@@ -3017,7 +3009,7 @@ int finput(T_SOC fd, char *s, int max) {
|
||||
}
|
||||
|
||||
// Like linput, but in memory (optimized)
|
||||
int binput(char *buff, char *s, int max) {
|
||||
int binput(const char *buff, char *s, int max) {
|
||||
int count = 0;
|
||||
int destCount = 0;
|
||||
|
||||
@@ -6020,6 +6012,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->warc_file, "");
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
opt->single_file = HTS_FALSE;
|
||||
opt->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
|
||||
22
src/htslib.h
22
src/htslib.h
@@ -199,6 +199,11 @@ T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
|
||||
etc.). */
|
||||
T_SOC newhttp_addr(httrackp *opt, const char *iadr, htsblk *retour, int port,
|
||||
int waitconnect, int addr_index, int *addr_count);
|
||||
/* Clips the formatted failure reason into r->msg, which also round-trips
|
||||
through the cache as X-StatusMessage. Leaves r->statuscode to the caller. */
|
||||
#define htsblk_failf(R, ...) \
|
||||
slprintfbuff_clip((R)->msg, sizeof((R)->msg), __VA_ARGS__)
|
||||
|
||||
HTS_INLINE void deletehttp(htsblk * r);
|
||||
HTS_INLINE int deleteaddr(htsblk * r);
|
||||
HTS_INLINE void deletesoc(T_SOC soc);
|
||||
@@ -262,7 +267,7 @@ HTS_INLINE void time_rfc822_local(char *s, struct tm *A);
|
||||
|
||||
HTS_INLINE int sendc(htsblk * r, const char *s);
|
||||
int finput(T_SOC fd, char *s, int max);
|
||||
int binput(char *buff, char *s, int max);
|
||||
int binput(const char *buff, char *s, int max);
|
||||
int linput(FILE * fp, char *s, int max);
|
||||
int linputsoc(T_SOC soc, char *s, int max);
|
||||
int linputsoc_t(T_SOC soc, char *s, int max, int timeout);
|
||||
@@ -607,6 +612,21 @@ static HTS_UNUSED size_t llint_to_size_t(LLint o) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Capacity for @p used bytes plus @p extra more plus @p slack spare;
|
||||
(size_t) -1 if the total exceeds (size_t) -2 or @p extra is negative
|
||||
(llint_to_size_t() would map that to a huge valid-looking size). */
|
||||
static HTS_UNUSED size_t llint_grow_size_t(size_t used, LLint extra,
|
||||
size_t slack) {
|
||||
const size_t max = (size_t) -2; /* (size_t) -1 is the error value */
|
||||
const size_t e = extra >= 0 ? llint_to_size_t(extra) : (size_t) -1;
|
||||
|
||||
if (e == (size_t) -1 || used > max || slack > max - used ||
|
||||
e > max - used - slack) {
|
||||
return (size_t) -1;
|
||||
}
|
||||
return used + e + slack;
|
||||
}
|
||||
|
||||
/* dirent() compatibility */
|
||||
#ifdef _WIN32
|
||||
/* Holds a UTF-8 d_name: MAX_PATH (260) UTF-16 units expand to <=3 bytes each.
|
||||
|
||||
@@ -547,6 +547,12 @@ struct httrackp {
|
||||
archive. Tail: ABI */
|
||||
hts_boolean warc_wacz; /**< --wacz: package archive+index+pages as a WACZ zip
|
||||
(implies --warc + --warc-cdx). Tail: ABI */
|
||||
hts_boolean single_file; /**< --single-file: once the mirror is done, rewrite
|
||||
each saved page with its assets inlined as
|
||||
data: URIs. Tail: ABI */
|
||||
LLint single_file_max_size; /**< --single-file-max-size: per-asset cap in
|
||||
bytes; a bigger asset stays a link.
|
||||
Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
|
||||
@@ -4583,7 +4583,7 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
/* seen as in error */
|
||||
in_error = back[b].r.statuscode;
|
||||
in_error_msg[0] = 0;
|
||||
strncat(in_error_msg, back[b].r.msg, sizeof(in_error_msg) - 1);
|
||||
strncatbuff(in_error_msg, back[b].r.msg, sizeof(in_error_msg) - 1);
|
||||
in_error_size = back[b].r.totalsize;
|
||||
/* don't break, even with "don't take error pages" switch, because we need to process the slot anyway (and cache the error) */
|
||||
}
|
||||
|
||||
@@ -173,8 +173,8 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
if (sscanf(line, "HTTP/%*d.%*d %d", &code) < 1)
|
||||
code = 0;
|
||||
if (code < 200 || code >= 300) {
|
||||
snprintf(retour->msg, sizeof(retour->msg), "proxy CONNECT refused: %s",
|
||||
strnotempty(line) ? line : "(no status)");
|
||||
htsblk_failf(retour, "proxy CONNECT refused: %s",
|
||||
strnotempty(line) ? line : "(no status)");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
145
src/htssafe.h
145
src/htssafe.h
@@ -33,6 +33,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#ifndef HTSSAFE_DEFH
|
||||
#define HTSSAFE_DEFH
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -116,6 +117,15 @@ static HTS_UNUSED void abortf_(const char *exp, const char *file, int line) {
|
||||
#endif
|
||||
#define HTS_IS_NOT_CHAR_BUFFER(VAR) (!HTS_IS_CHAR_BUFFER(VAR))
|
||||
|
||||
/* Source capacity for the buff() family, (size_t)-1 when unknown; sizeof of the
|
||||
TYPE keeps a decayed operand ("buf + 1") off -Wsizeof-array-decay. */
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
#define HTS_SIZEOF_SRC_(B) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(__typeof__(B)))
|
||||
#else
|
||||
#define HTS_SIZEOF_SRC_(B) (HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B))
|
||||
#endif
|
||||
|
||||
/* Compile-time checks. */
|
||||
static HTS_UNUSED void htssafe_compile_time_check_(void) {
|
||||
char array[32];
|
||||
@@ -205,19 +215,17 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
|
||||
#define strncatbuff(A, B, N) \
|
||||
__builtin_choose_expr( \
|
||||
HTS_IS_CHAR_BUFFER(A), \
|
||||
strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), N, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__), \
|
||||
strncatbuff_ptr_((A), (B), (N)))
|
||||
__builtin_choose_expr(HTS_IS_CHAR_BUFFER(A), \
|
||||
strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), N, \
|
||||
"overflow while appending '" #B \
|
||||
"' to '" #A "'", \
|
||||
__FILE__, __LINE__), \
|
||||
strncatbuff_ptr_((A), (B), (N)))
|
||||
#else
|
||||
#define strncatbuff(A, B, N) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(A) \
|
||||
? strncat(A, B, N) \
|
||||
: strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), N, \
|
||||
: strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), N, \
|
||||
"overflow while appending '" #B "' to '" #A "'", \
|
||||
__FILE__, __LINE__))
|
||||
#endif
|
||||
@@ -232,9 +240,7 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#define strcatbuff(A, B) \
|
||||
__builtin_choose_expr( \
|
||||
HTS_IS_CHAR_BUFFER(A), \
|
||||
strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
(size_t) -1, \
|
||||
strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), (size_t) -1, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__), \
|
||||
strcatbuff_ptr_((A), (B)))
|
||||
@@ -242,9 +248,7 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#define strcatbuff(A, B) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(A) \
|
||||
? strcat(A, B) \
|
||||
: strncat_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
(size_t) -1, \
|
||||
: strncat_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), (size_t) -1, \
|
||||
"overflow while appending '" #B "' to '" #A "'", \
|
||||
__FILE__, __LINE__))
|
||||
#endif
|
||||
@@ -257,19 +261,17 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
|
||||
#define strcpybuff(A, B) \
|
||||
__builtin_choose_expr( \
|
||||
HTS_IS_CHAR_BUFFER(A), \
|
||||
strcpy_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
"overflow while copying '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__), \
|
||||
strcpybuff_ptr_((A), (B)))
|
||||
__builtin_choose_expr(HTS_IS_CHAR_BUFFER(A), \
|
||||
strcpy_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), \
|
||||
"overflow while copying '" #B "' to '" #A \
|
||||
"'", \
|
||||
__FILE__, __LINE__), \
|
||||
strcpybuff_ptr_((A), (B)))
|
||||
#else
|
||||
#define strcpybuff(A, B) \
|
||||
(HTS_IS_NOT_CHAR_BUFFER(A) \
|
||||
? strcpy(A, B) \
|
||||
: strcpy_safe_(A, sizeof(A), B, \
|
||||
HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
: strcpy_safe_(A, sizeof(A), B, HTS_SIZEOF_SRC_(B), \
|
||||
"overflow while copying '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__))
|
||||
#endif
|
||||
@@ -286,24 +288,24 @@ static char *strncatbuff_ptr_(char *dest, const char *src, size_t n) {
|
||||
* Append characters of "B" to "A", "A" having a maximum capacity of "S".
|
||||
*/
|
||||
#define strlcatbuff(A, B, S) \
|
||||
strncat_safe_(A, S, B, HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
(size_t) -1, "overflow while appending '" #B "' to '" #A "'", \
|
||||
__FILE__, __LINE__)
|
||||
strncat_safe_(A, S, B, HTS_SIZEOF_SRC_(B), (size_t) -1, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__)
|
||||
|
||||
/**
|
||||
* Append at most "N" characters of "B" to "A", "A" having a maximum capacity
|
||||
* of "S".
|
||||
*/
|
||||
#define strlncatbuff(A, B, S, N) \
|
||||
strncat_safe_(A, S, B, HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
N, "overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
strncat_safe_(A, S, B, HTS_SIZEOF_SRC_(B), N, \
|
||||
"overflow while appending '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__)
|
||||
|
||||
/**
|
||||
* Copy characters of "B" to "A", "A" having a maximum capacity of "S".
|
||||
*/
|
||||
#define strlcpybuff(A, B, S) \
|
||||
strcpy_safe_(A, S, B, HTS_IS_NOT_CHAR_BUFFER(B) ? (size_t) -1 : sizeof(B), \
|
||||
strcpy_safe_(A, S, B, HTS_SIZEOF_SRC_(B), \
|
||||
"overflow while copying '" #B "' to '" #A "'", __FILE__, \
|
||||
__LINE__)
|
||||
|
||||
@@ -422,7 +424,9 @@ static HTS_INLINE HTS_UNUSED htsbuff htsbuff_ptr_(char *buf, size_t cap) {
|
||||
*/
|
||||
static HTS_INLINE HTS_UNUSED void htsbuff_catn(htsbuff *b, const char *s,
|
||||
size_t n) {
|
||||
const size_t add = strnlen(s, n);
|
||||
/* the (size_t)-1 "no limit" sentinel would reach strnlen as a bound past
|
||||
PTRDIFF_MAX */
|
||||
const size_t add = n != (size_t) -1 ? strnlen(s, n) : strlen(s);
|
||||
/* Overflow-safe: keep the (potentially huge) 'add' alone on one side. The
|
||||
maintained invariant len < cap makes 'cap - len' >= 1 (no underflow), so
|
||||
'add < cap - len' cannot wrap the way 'len + add < cap' could. */
|
||||
@@ -456,6 +460,85 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
|
||||
return b->buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy src into dest (capacity size, NUL included), truncating to fit and
|
||||
* always NUL-terminating. Unlike strlcpybuff() it never aborts, so it suits a
|
||||
* value read back from a cache, a header or the wire, where refusing the whole
|
||||
* record is worse than keeping a clipped one. Returns HTS_TRUE if it all fit;
|
||||
* callers that clip on purpose ignore that, so it is not HTS_CHECK_RESULT.
|
||||
*/
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean strclipbuff(char *dest, size_t size,
|
||||
const char *src) {
|
||||
size_t len, copy;
|
||||
|
||||
assertf(dest != NULL && src != NULL && size != 0);
|
||||
len = strlen(src);
|
||||
copy = len < size ? len : size - 1;
|
||||
memcpy(dest, src, copy);
|
||||
dest[copy] = '\0';
|
||||
return copy == len ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Callers that deliberately ignore truncation use this instead of
|
||||
* slprintfbuff(), so it is not HTS_CHECK_RESULT.
|
||||
*/
|
||||
static HTS_INLINE HTS_UNUSED HTS_PRINTF_FUN(3, 0) hts_boolean
|
||||
vslprintfbuff(char *dest, size_t size, const char *fmt, va_list args) {
|
||||
int ret;
|
||||
|
||||
assertf(dest != NULL && size != 0);
|
||||
/* a vsnprintf failing outright may write nothing at all, leaving whatever
|
||||
the caller had on the stack for it to publish */
|
||||
dest[0] = '\0';
|
||||
ret = vsnprintf(dest, size, fmt, args);
|
||||
/* pre-C99 runtimes (msvcrt _vsnprintf) return -1 and do not terminate */
|
||||
dest[size - 1] = '\0';
|
||||
return ret >= 0 && (size_t) ret < size ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Formatted print into dest (capacity size, NUL included), truncating to fit
|
||||
* and always NUL-terminating. Returns HTS_TRUE if the whole output fit; the
|
||||
* result is the only truncation signal, so it must be acted on. Unlike
|
||||
* strcpybuff() it never aborts, so it suits text built from remote input.
|
||||
*/
|
||||
static HTS_INLINE HTS_UNUSED HTS_CHECK_RESULT HTS_PRINTF_FUN(3, 4) hts_boolean
|
||||
slprintfbuff(char *dest, size_t size, const char *fmt, ...) {
|
||||
va_list args;
|
||||
hts_boolean ret;
|
||||
|
||||
va_start(args, fmt);
|
||||
ret = vslprintfbuff(dest, size, fmt, args);
|
||||
va_end(args);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* slprintfbuff() for diagnostics quoting remote or client text, which are
|
||||
* meant to be clipped: nothing to act on, hence not HTS_CHECK_RESULT. A (void)
|
||||
* cast on slprintfbuff() is no substitute, GCC warns through it.
|
||||
*/
|
||||
static HTS_INLINE HTS_UNUSED HTS_PRINTF_FUN(3, 4) void slprintfbuff_clip(
|
||||
char *dest, size_t size, const char *fmt, ...) {
|
||||
va_list args;
|
||||
|
||||
va_start(args, fmt);
|
||||
(void) vslprintfbuff(dest, size, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
/**
|
||||
* slprintfbuff() over the in-scope array ARR (capacity = sizeof(ARR)).
|
||||
* On GCC/Clang a pointer is a compile error; use slprintfbuff() for those.
|
||||
*/
|
||||
#if (defined(__GNUC__) && !defined(__cplusplus))
|
||||
#define sprintfbuff(ARR, ...) \
|
||||
slprintfbuff((ARR), sizeof(ARR) + htsbuff_must_be_array_(ARR), __VA_ARGS__)
|
||||
#else
|
||||
#define sprintfbuff(ARR, ...) slprintfbuff((ARR), sizeof(ARR), __VA_ARGS__)
|
||||
#endif
|
||||
|
||||
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
|
||||
added bounds checking. freet() also NULLs the freed pointer and tolerates
|
||||
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the
|
||||
|
||||
1025
src/htsselftest.c
1025
src/htsselftest.c
File diff suppressed because it is too large
Load Diff
206
src/htsserver.c
206
src/htsserver.c
@@ -318,6 +318,18 @@ typedef struct {
|
||||
error_redirect = "/server/error.html"; \
|
||||
} while(0)
|
||||
|
||||
/* Longest error message shown on the error page; the rest is clipped. */
|
||||
#define ERROR_MESSAGE_MAX 1024
|
||||
|
||||
/* SET_ERROR() with a printf format. Clips: these messages quote posted fields,
|
||||
whose length the client picks. */
|
||||
#define SET_ERRORF(...) \
|
||||
do { \
|
||||
char errbuf[ERROR_MESSAGE_MAX]; \
|
||||
slprintfbuff_clip(errbuf, sizeof(errbuf), __VA_ARGS__); \
|
||||
SET_ERROR(errbuf); \
|
||||
} while (0)
|
||||
|
||||
/* Longest "sid" value worth unescaping: the expected one is an md5 hex digest,
|
||||
so anything near this is already invalid and is rejected unread. */
|
||||
#define SID_VALUE_MAX 64
|
||||
@@ -363,6 +375,59 @@ static hts_boolean body_sid_is_valid(const char *body, const char *expected) {
|
||||
return seen;
|
||||
}
|
||||
|
||||
/** Append src to the NUL-terminated dst of capacity size (NUL included).
|
||||
False, leaving dst untouched, if it would not fit: unlike strcatbuff() this
|
||||
never aborts, because every piece appended here is client-supplied. */
|
||||
static hts_boolean path_append(char *dst, size_t size, const char *src) {
|
||||
const size_t used = strlen(dst);
|
||||
const size_t len = strlen(src);
|
||||
|
||||
/* dst holds at most size-1 bytes, so "size - used" is >= 1 and the untrusted
|
||||
len stays alone: "used + len < size" could wrap and pass. */
|
||||
if (len >= size - used) {
|
||||
return HTS_FALSE;
|
||||
}
|
||||
memcpy(dst + used, src, len + 1);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Append c to dst as an HTML entity, or return HTS_FALSE if it needs none. */
|
||||
static hts_boolean cat_html_escaped(String *dst, char c) {
|
||||
switch (c) {
|
||||
case '<':
|
||||
StringCat(*dst, "<");
|
||||
break;
|
||||
case '>':
|
||||
StringCat(*dst, ">");
|
||||
break;
|
||||
case '&':
|
||||
StringCat(*dst, "&");
|
||||
break;
|
||||
case '\'':
|
||||
StringCat(*dst, "'");
|
||||
break;
|
||||
default:
|
||||
return HTS_FALSE;
|
||||
}
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Append the value of a double-quoted command-line argument: escaped for HTML,
|
||||
which the browser undoes when it posts the command line back, and for the
|
||||
argv splitter, which does not. */
|
||||
static void cat_cmdline_arg(String *output, const char *value) {
|
||||
const char *a;
|
||||
|
||||
for (a = value; *a != '\0'; a++) {
|
||||
if (*a == '\\' || *a == '\"') {
|
||||
StringCat(*output, "\\");
|
||||
}
|
||||
if (!cat_html_escaped(output, *a)) {
|
||||
StringMemcat(*output, a, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
int timeout = 30;
|
||||
int retour = 0;
|
||||
@@ -374,6 +439,9 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
String tmpbuff = STRING_EMPTY;
|
||||
String tmpbuff2 = STRING_EMPTY;
|
||||
String fspath = STRING_EMPTY;
|
||||
/* Project directory this server set up; the only root /website/ serves from,
|
||||
and deliberately not cleared between requests. */
|
||||
String website = STRING_EMPTY;
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
|
||||
/* Load strings */
|
||||
@@ -669,7 +737,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
fp = fopen(StringBuff(fspath), "rb");
|
||||
if (fp) {
|
||||
/* Read file */
|
||||
while(!feof(fp)) {
|
||||
while (!feof(fp) && !ferror(fp)) {
|
||||
char *str = line;
|
||||
char *pos;
|
||||
|
||||
@@ -785,6 +853,11 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
if (!structcheck(StringBuff(tmpbuff))) {
|
||||
FILE *fp;
|
||||
|
||||
/* Both halves of fspath come from posted fields, so a ".."
|
||||
in them would escape the mirror once served. */
|
||||
if (strstr(StringBuff(fspath), "..") == NULL) {
|
||||
StringCopy(website, StringBuff(fspath));
|
||||
}
|
||||
StringCat(tmpbuff, "winprofile.ini");
|
||||
fp = fopen(StringBuff(tmpbuff), "wb");
|
||||
if (fp != NULL) {
|
||||
@@ -818,28 +891,18 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
commandEnd = 1;
|
||||
}
|
||||
} else {
|
||||
char tmp[1024];
|
||||
|
||||
sprintf(tmp,
|
||||
"Unable to write %d bytes in the the init file %s",
|
||||
count, StringBuff(fspath));
|
||||
SET_ERROR(tmp);
|
||||
SET_ERRORF(
|
||||
"Unable to write %d bytes in the the init file %s",
|
||||
count, StringBuff(fspath));
|
||||
}
|
||||
fclose(fp);
|
||||
} else {
|
||||
char tmp[1024];
|
||||
|
||||
sprintf(tmp, "Unable to create the init file %s",
|
||||
StringBuff(fspath));
|
||||
SET_ERROR(tmp);
|
||||
SET_ERRORF("Unable to create the init file %s",
|
||||
StringBuff(fspath));
|
||||
}
|
||||
} else {
|
||||
char tmp[1024];
|
||||
|
||||
sprintf(tmp,
|
||||
"Unable to create the directory structure in %s",
|
||||
StringBuff(fspath));
|
||||
SET_ERROR(tmp);
|
||||
SET_ERRORF("Unable to create the directory structure in %s",
|
||||
StringBuff(fspath));
|
||||
}
|
||||
|
||||
} else {
|
||||
@@ -857,7 +920,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
|
||||
/* Response */
|
||||
if (meth) {
|
||||
int virtualpath = 0;
|
||||
hts_boolean virtualpath = HTS_FALSE;
|
||||
char *pos;
|
||||
char *url = strchr(line1, ' ');
|
||||
|
||||
@@ -868,11 +931,11 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
char *qpos;
|
||||
|
||||
/* get the URL */
|
||||
fsfile[0] = '\0';
|
||||
if (error_redirect == NULL) {
|
||||
if ((qpos = strchr(url, '?'))) {
|
||||
*qpos = '\0';
|
||||
}
|
||||
fsfile[0] = '\0';
|
||||
if (strcmp(url, "/") == 0) {
|
||||
file = "/server/index.html";
|
||||
meth = 2;
|
||||
@@ -885,7 +948,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
|
||||
if (strncmp(file, "/website/", 9) == 0) {
|
||||
virtualpath = 1;
|
||||
virtualpath = HTS_TRUE;
|
||||
}
|
||||
|
||||
/* override */
|
||||
@@ -899,20 +962,28 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
}
|
||||
|
||||
if (strlen(path) + strlen(file) + 32 < sizeof(fsfile)) {
|
||||
if (strncmp(file, "/website/", 9) != 0) {
|
||||
sprintf(fsfile, "%shtml%s", path, file);
|
||||
} else {
|
||||
intptr_t adr = 0;
|
||||
/* the override above may have swapped a mirror path for a GUI page */
|
||||
virtualpath = strncmp(file, "/website/", 9) == 0;
|
||||
|
||||
if (coucal_readptr(NewLangList, "projpath", &adr)) {
|
||||
sprintf(fsfile, "%s%s", (char *) adr, file + 9);
|
||||
}
|
||||
if (!virtualpath) {
|
||||
if (!path_append(fsfile, sizeof(fsfile), path) ||
|
||||
!path_append(fsfile, sizeof(fsfile), "html") ||
|
||||
!path_append(fsfile, sizeof(fsfile), file)) {
|
||||
fsfile[0] = '\0';
|
||||
}
|
||||
} else if (StringNotEmpty(website)) {
|
||||
/* Never the posted "projpath": a client root reads any file. */
|
||||
if (!path_append(fsfile, sizeof(fsfile), StringBuff(website)) ||
|
||||
!path_append(fsfile, sizeof(fsfile), "/") ||
|
||||
!path_append(fsfile, sizeof(fsfile), file + 9)) {
|
||||
fsfile[0] = '\0';
|
||||
}
|
||||
}
|
||||
|
||||
if (fsfile[0] && strstr(file, "..") == NULL
|
||||
&& (fp = fopen(fsfile, "rb"))) {
|
||||
/* Regular files only: reading a directory or FIFO never ends, and
|
||||
"path" may hold "..", so only the untrusted halves are checked. */
|
||||
if (fsfile[0] && strstr(file, "..") == NULL && fexist(fsfile) &&
|
||||
(fp = fopen(fsfile, "rb"))) {
|
||||
char ok[] =
|
||||
"HTTP/1.0 200 OK\r\n" "Connection: close\r\n"
|
||||
"Server: httrack-small-server\r\n" "Content-type: text/html\r\n"
|
||||
@@ -965,11 +1036,13 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
StringCat(headers, "\r\n");
|
||||
}
|
||||
coucal_write(NewLangList, "redirect", (intptr_t) NULL);
|
||||
} else if (is_html(file)) {
|
||||
} else if (!virtualpath && is_html(file)) {
|
||||
/* GUI templates only: ${_sid} in a mirrored page would hand the
|
||||
crawled site the session id that authenticates commands */
|
||||
int outputmode = 0;
|
||||
|
||||
StringMemcat(headers, ok, sizeof(ok) - 1);
|
||||
while(!feof(fp)) {
|
||||
while (!feof(fp) && !ferror(fp)) {
|
||||
char *str = line;
|
||||
int prevlen = (int) StringLength(output);
|
||||
int nocr = 0;
|
||||
@@ -993,6 +1066,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
int p;
|
||||
int format = 0;
|
||||
int listDefault = 0;
|
||||
hts_boolean unquoted = HTS_FALSE;
|
||||
|
||||
name[0] = '\0';
|
||||
strlncatbuff(name, str, sizeof(name_), n);
|
||||
@@ -1002,6 +1076,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
} else if ((p = strfield(name, "html:"))) {
|
||||
name += p;
|
||||
format = 1;
|
||||
} else if ((p = strfield(name, "unquoted:"))) {
|
||||
name += p;
|
||||
unquoted = HTS_TRUE;
|
||||
} else if ((p = strfield(name, "arg:"))) {
|
||||
name += p;
|
||||
format = 5;
|
||||
} else if ((p = strfield(name, "list:"))) {
|
||||
name += p;
|
||||
format = 2;
|
||||
@@ -1138,8 +1218,8 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
test:<if ==0>:<if ==1>:<if == 2>..
|
||||
ztest:<if == 0 || !exist>:<if == 1>:<if == 2>..
|
||||
*/
|
||||
else if ((p = strfield(name, "test:"))
|
||||
|| (p = strfield(name, "ztest:"))) {
|
||||
else if ((p = strfield(name, "test:")) ||
|
||||
(p = strfield(name, "ztest:"))) {
|
||||
intptr_t adr = 0;
|
||||
char *pos2;
|
||||
int ztest = (name[0] == 'z');
|
||||
@@ -1242,6 +1322,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
}
|
||||
}
|
||||
/* consumed here: it shares nothing with the list and
|
||||
option formats below */
|
||||
if (format == 5 && langstr != NULL && outputmode != -1) {
|
||||
cat_cmdline_arg(&output, langstr);
|
||||
langstr = NULL;
|
||||
}
|
||||
if (langstr && outputmode != -1) {
|
||||
switch (format) {
|
||||
case 0:
|
||||
@@ -1259,18 +1345,18 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
StringMemcat(output, &c, 1);
|
||||
}
|
||||
a += 2;
|
||||
} else if (outputmode && a[0] == '<') {
|
||||
StringCat(output, "<");
|
||||
} else if (outputmode && a[0] == '>') {
|
||||
StringCat(output, ">");
|
||||
} else if (outputmode && a[0] == '&') {
|
||||
StringCat(output, "&");
|
||||
} else if (outputmode && a[0] == '\'') {
|
||||
StringCat(output, "'");
|
||||
} else if (unquoted && a[0] == '\"') {
|
||||
/* the browser posts an entity back as a raw
|
||||
quote, which would open a quoted run in the
|
||||
argv splitter; a URI cannot hold one anyway */
|
||||
StringCat(output, "%22");
|
||||
} else if (outputmode &&
|
||||
cat_html_escaped(&output, a[0])) {
|
||||
/* appended as an entity */
|
||||
} else if (outputmode == 3 && a[0] == ' ') {
|
||||
StringCat(output, "%20");
|
||||
} else if (outputmode >= 2
|
||||
&& ((unsigned char) a[0]) < 32) {
|
||||
} else if (outputmode >= 2 &&
|
||||
((unsigned char) a[0]) < 32) {
|
||||
char tmp[32];
|
||||
|
||||
sprintf(tmp, "%%%02x", (unsigned char) a[0]);
|
||||
@@ -1331,20 +1417,10 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
StringClear(tmpbuff);
|
||||
break;
|
||||
case '<':
|
||||
StringCat(tmpbuff, "<");
|
||||
break;
|
||||
case '>':
|
||||
StringCat(tmpbuff, ">");
|
||||
break;
|
||||
case '&':
|
||||
StringCat(tmpbuff, "&");
|
||||
break;
|
||||
case '\'':
|
||||
StringCat(tmpbuff, "'");
|
||||
break;
|
||||
default:
|
||||
StringMemcat(tmpbuff, fstr, 1);
|
||||
if (!cat_html_escaped(&tmpbuff, *fstr)) {
|
||||
StringMemcat(tmpbuff, fstr, 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
fstr++;
|
||||
@@ -1384,7 +1460,9 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
if (is_text(file)) {
|
||||
if (is_html(file)) {
|
||||
StringMemcat(headers, ok, sizeof(ok) - 1);
|
||||
} else if (is_text(file)) {
|
||||
StringMemcat(headers, ok_text, sizeof(ok_text) - 1);
|
||||
} else if (is_js(file)) {
|
||||
StringMemcat(headers, ok_js, sizeof(ok_js) - 1);
|
||||
@@ -1398,14 +1476,15 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
while(!feof(fp)) {
|
||||
int n = (int) fread(line, 1, sizeof(line) - 2, fp);
|
||||
|
||||
if (n > 0) {
|
||||
StringMemcat(output, line, n);
|
||||
if (n <= 0) {
|
||||
break; /* short read: EOF or error, never a retry */
|
||||
}
|
||||
StringMemcat(output, line, n);
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
} else if (strcmp(file, "/ping") == 0
|
||||
|| strncmp(file, "/ping?", 6) == 0) {
|
||||
} else if (strcmp(file, "/ping") == 0 ||
|
||||
strncmp(file, "/ping?", 6) == 0) {
|
||||
char error_hdr[] =
|
||||
"HTTP/1.0 200 Pong\r\n" "Server: httrack small server\r\n"
|
||||
"Content-type: text/html\r\n";
|
||||
@@ -1491,6 +1570,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
StringFree(tmpbuff);
|
||||
StringFree(tmpbuff2);
|
||||
StringFree(fspath);
|
||||
StringFree(website);
|
||||
|
||||
if (buffer)
|
||||
free(buffer);
|
||||
|
||||
1160
src/htssinglefile.c
Normal file
1160
src/htssinglefile.c
Normal file
File diff suppressed because it is too large
Load Diff
84
src/htssinglefile.h
Normal file
84
src/htssinglefile.h
Normal file
@@ -0,0 +1,84 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* --single-file: post-mirror pass embedding each page's assets as data: URIs.
|
||||
Internal, not installed. Unlike -%M (one MHT archive for the whole mirror)
|
||||
it rewrites the saved pages in place and keeps page-to-page links relative,
|
||||
so the mirror stays browsable and every page also stands alone.
|
||||
Limitation: an inlined stylesheet becomes a data: URL, whose path is opaque,
|
||||
so an asset inside it that stayed a link (over the cap) no longer resolves.
|
||||
Raise --single-file-max-size past it to inline it too. */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_SINGLEFILE_DEFH
|
||||
#define HTS_SINGLEFILE_DEFH
|
||||
|
||||
#include "htsopt.h"
|
||||
#include "htsstrings.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* --single-file-max-size default: a bigger asset stays an ordinary link. */
|
||||
#define SINGLEFILE_DEFAULT_MAX_SIZE (10 * 1024 * 1024)
|
||||
|
||||
/* Never base64 more than this, whatever the cap: code64() sizes in int. */
|
||||
#define SINGLEFILE_HARD_MAX_SIZE (256 * 1024 * 1024)
|
||||
|
||||
/* Total bytes one page may inline. Nested @import fans out multiplicatively,
|
||||
so a few hundred bytes of hostile CSS can otherwise ask for gigabytes. */
|
||||
#define SINGLEFILE_MAX_PAGE_SIZE (64 * 1024 * 1024)
|
||||
|
||||
/* Rewrite every HTML page the mirror produced. No-op unless opt->single_file;
|
||||
call once the tree is final, after the update purge. */
|
||||
void singlefile_process_mirror(httrackp *opt);
|
||||
|
||||
/* Rewrite one HTML document held in memory, appending the result to out.
|
||||
root is the mirror directory that references may not escape; page_path is
|
||||
the document's own path under it (both UTF-8, '/' or native separators).
|
||||
page_budget caps the total inlined bytes, since nested @import fans out
|
||||
multiplicatively; the mirror pass passes SINGLEFILE_MAX_PAGE_SIZE.
|
||||
Returns HTS_TRUE if at least one reference was replaced; out may still
|
||||
differ from the input when that is HTS_FALSE, since a style or srcset value
|
||||
is re-serialized in place. */
|
||||
hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
|
||||
const char *page_path, const char *html,
|
||||
size_t html_len, LLint page_budget,
|
||||
String *out);
|
||||
|
||||
/* singlefile_rewrite_html() over the file at page_path, rewritten in place.
|
||||
Returns HTS_TRUE if the file changed. */
|
||||
hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
|
||||
const char *page_path);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
63
src/htsstats.h
Normal file
63
src/htsstats.h
Normal file
@@ -0,0 +1,63 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: in-progress display rows, shared by httrack and htsserver .h */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_STATS_DEFH
|
||||
#define HTS_STATS_DEFH
|
||||
|
||||
#include "htsglobal.h"
|
||||
|
||||
/* One row of the "in progress" display, shared so the CLI (httrack) and the
|
||||
web GUI (htsserver) cannot drift apart: each fills its own array. */
|
||||
|
||||
#define NStatsBuffer 14
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_t_StatsBuffer
|
||||
#define HTS_DEF_FWSTRUCT_t_StatsBuffer
|
||||
typedef struct t_StatsBuffer t_StatsBuffer;
|
||||
#endif
|
||||
struct t_StatsBuffer {
|
||||
char name[1024];
|
||||
char file[1024];
|
||||
char state[288]; // a short label plus back->info[256]
|
||||
char BIGSTK url_sav[HTS_URLMAXSIZE * 2]; // pour cancel
|
||||
char BIGSTK url_adr[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK url_fil[HTS_URLMAXSIZE * 2];
|
||||
LLint size;
|
||||
LLint sizetot;
|
||||
int offset;
|
||||
//
|
||||
int back;
|
||||
//
|
||||
int actived; // pour disabled
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -308,8 +308,10 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
// copy only the current path
|
||||
curr = _curr;
|
||||
strlcpybuff(curr, curr_fil, sizeof(_curr));
|
||||
if ((a = strchr(curr, '?')) == NULL) // couper au ? (params)
|
||||
a = curr + strlen(curr) - 1; // pas de params: aller à la fin
|
||||
if ((a = strchr(curr, '?')) == NULL) { // cut at the ? (query parameters)
|
||||
// an empty path has no last character: curr-1 would read before the buffer
|
||||
a = curr[0] != '\0' ? curr + strlen(curr) - 1 : curr;
|
||||
}
|
||||
while((*a != '/') && (a > curr))
|
||||
a--; // chercher dernier / du chemin courant
|
||||
if (*a == '/')
|
||||
@@ -1318,19 +1320,19 @@ HTSEXT_API hts_boolean hts_findnext(find_handle find) {
|
||||
if (find) {
|
||||
#ifdef _WIN32
|
||||
if ((FindNextFileA(find->handle, &find->hdata)))
|
||||
return 1;
|
||||
return HTS_TRUE;
|
||||
#else
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
|
||||
memset(&(find->filestat), 0, sizeof(find->filestat));
|
||||
if ((find->dirp = readdir(find->hdir)))
|
||||
if (find->dirp->d_name)
|
||||
if (!STAT
|
||||
(concat(catbuff, sizeof(catbuff), find->path, find->dirp->d_name), &find->filestat))
|
||||
return 1;
|
||||
if (!STAT(
|
||||
concat(catbuff, sizeof(catbuff), find->path, find->dirp->d_name),
|
||||
&find->filestat))
|
||||
return HTS_TRUE;
|
||||
#endif
|
||||
}
|
||||
return 0;
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
HTSEXT_API int hts_findclose(find_handle find) {
|
||||
|
||||
45
src/htsweb.c
45
src/htsweb.c
@@ -62,6 +62,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsmd5.c"
|
||||
#include "md5.c"
|
||||
|
||||
#include "htscmdline.h"
|
||||
#include "htsserver.h"
|
||||
#include "htsurlport.h"
|
||||
#include "htsweb.h"
|
||||
@@ -319,10 +320,8 @@ int main(int argc, char *argv[]) {
|
||||
static int webhttrack_runmain(httrackp * opt, int argc, char **argv);
|
||||
static void back_launch_cmd(void *pP) {
|
||||
char *cmd = (char *) pP;
|
||||
char **argv = (char **) malloct(1024 * sizeof(char *));
|
||||
char **argv;
|
||||
int argc = 0;
|
||||
int i = 0;
|
||||
int g = 0;
|
||||
|
||||
//
|
||||
httrackp *opt;
|
||||
@@ -333,28 +332,19 @@ static void back_launch_cmd(void *pP) {
|
||||
commandReturnCmdl = strdup(cmd);
|
||||
|
||||
/* split */
|
||||
argv[0] = strdup("webhttrack");
|
||||
argv[1] = cmd;
|
||||
argc++;
|
||||
i = 0;
|
||||
while(cmd[i]) {
|
||||
if (cmd[i] == '\t' || cmd[i] == '\r' || cmd[i] == '\n') {
|
||||
cmd[i] = ' ';
|
||||
}
|
||||
i++;
|
||||
}
|
||||
i = 0;
|
||||
while(cmd[i]) {
|
||||
if (cmd[i] == '\"')
|
||||
g = !g;
|
||||
if (cmd[i] == ' ') {
|
||||
if (!g) {
|
||||
cmd[i] = '\0';
|
||||
argv[argc++] = cmd + i + 1;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
argv = hts_split_cmdline(cmd, &argc);
|
||||
if (argv == NULL) {
|
||||
if (commandReturnMsg)
|
||||
free(commandReturnMsg);
|
||||
commandReturnMsg = strdup("could not parse the command line");
|
||||
commandReturn = -1;
|
||||
commandRunning = 0;
|
||||
commandEnd = 1;
|
||||
free(cmd);
|
||||
return;
|
||||
}
|
||||
/* drop the program name the posted command line carries */
|
||||
argv[0] = strdupt("webhttrack");
|
||||
|
||||
/* init */
|
||||
hts_init();
|
||||
@@ -383,6 +373,7 @@ static void back_launch_cmd(void *pP) {
|
||||
|
||||
/* free */
|
||||
free(cmd);
|
||||
freet(argv[0]);
|
||||
freet(argv);
|
||||
return;
|
||||
}
|
||||
@@ -682,10 +673,10 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
|
||||
char *eps = strchr(back[i].url_adr, '/');
|
||||
int count;
|
||||
|
||||
if (ep != NULL && ep < eps
|
||||
&& (count = (int) (ep - back[i].url_adr)) < 4) {
|
||||
if (ep != NULL && eps != NULL && ep < eps &&
|
||||
(count = (int) (ep - back[i].url_adr)) < 4) {
|
||||
proto[0] = '\0';
|
||||
strncat(proto, back[i].url_adr, count);
|
||||
strncatbuff(proto, back[i].url_adr, count);
|
||||
}
|
||||
}
|
||||
snprintf(StatsBuffer[index].state, sizeof(StatsBuffer[index].state),
|
||||
|
||||
18
src/htsweb.h
18
src/htsweb.h
@@ -35,26 +35,10 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htsglobal.h"
|
||||
#include "htscore.h"
|
||||
#include "htsstats.h"
|
||||
|
||||
#define NStatsBuffer 14
|
||||
#define MAX_LEN_INPROGRESS 40
|
||||
|
||||
typedef struct t_StatsBuffer {
|
||||
char name[1024];
|
||||
char file[1024];
|
||||
char state[256];
|
||||
char url_sav[HTS_URLMAXSIZE * 2]; // pour cancel
|
||||
char url_adr[HTS_URLMAXSIZE * 2];
|
||||
char url_fil[HTS_URLMAXSIZE * 2];
|
||||
LLint size;
|
||||
LLint sizetot;
|
||||
int offset;
|
||||
//
|
||||
int back;
|
||||
//
|
||||
int actived; // pour disabled
|
||||
} t_StatsBuffer;
|
||||
|
||||
typedef struct t_InpInfo {
|
||||
int ask_refresh;
|
||||
int refresh;
|
||||
|
||||
@@ -46,6 +46,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "httrack.h"
|
||||
#include "htslib.h"
|
||||
#include "htscharset.h" // after htslib.h: winsock2.h must precede windows.h
|
||||
#include "htsbacktrace.h"
|
||||
|
||||
/* Static definitions */
|
||||
static int fexist(const char *s);
|
||||
@@ -72,10 +73,6 @@ static int linput(FILE * fp, char *s, int max);
|
||||
#include <sys/ioctl.h>
|
||||
#endif
|
||||
#include <ctype.h>
|
||||
#if (defined(__linux) && defined(HAVE_EXECINFO_H))
|
||||
#include <execinfo.h>
|
||||
#define USES_BACKTRACE
|
||||
#endif
|
||||
/* END specific definitions */
|
||||
|
||||
static void __cdecl htsshow_init(t_hts_callbackarg * carg);
|
||||
@@ -233,8 +230,7 @@ static hts_boolean vt_size_refresh(void) {
|
||||
*/
|
||||
#define STYLE_STATVALUES VT_BOLD
|
||||
#define STYLE_STATTEXT VT_UNBOLD
|
||||
#define STYLE_STATRESET VT_UNBOLD
|
||||
#define NStatsBuffer 14
|
||||
#define STYLE_STATRESET VT_UNBOLD
|
||||
/* Rows the stats block and "Current job" take above the in-progress list. */
|
||||
#define NStatsHeaderRows 7
|
||||
|
||||
@@ -880,21 +876,6 @@ static void sig_doback(int blind) { // mettre en backing
|
||||
#undef FD_ERR
|
||||
#define FD_ERR 2
|
||||
|
||||
static void print_backtrace(void) {
|
||||
#ifdef USES_BACKTRACE
|
||||
void *stack[256];
|
||||
const int size = backtrace(stack, sizeof(stack)/sizeof(stack[0]));
|
||||
if (size != 0) {
|
||||
backtrace_symbols_fd(stack, size, FD_ERR);
|
||||
}
|
||||
#else
|
||||
const char msg[] = "No stack trace available on this OS :(\n";
|
||||
if (write(FD_ERR, msg, sizeof(msg) - 1) != sizeof(msg) - 1) {
|
||||
/* sorry GCC */
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static size_t print_num(char *buffer, int num) {
|
||||
size_t i, j;
|
||||
if (num < 0) {
|
||||
@@ -928,7 +909,7 @@ static void sig_fatal(int code) {
|
||||
size += print_num(&buffer[size], code);
|
||||
buffer[size++] = '\n';
|
||||
(void) (write(FD_ERR, buffer, size) == size);
|
||||
print_backtrace();
|
||||
hts_print_backtrace(FD_ERR);
|
||||
(void) (write(FD_ERR, msgreport, sizeof(msgreport) - 1)
|
||||
== sizeof(msgreport) - 1);
|
||||
abort();
|
||||
@@ -951,6 +932,7 @@ static void sig_leave(int code) {
|
||||
}
|
||||
|
||||
static void signal_handlers(void) {
|
||||
hts_backtrace_init();
|
||||
#ifdef _WIN32
|
||||
signal(SIGINT, sig_leave); // ^C
|
||||
signal(SIGTERM, sig_finish); // kill <process>
|
||||
|
||||
@@ -36,26 +36,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsglobal.h"
|
||||
#include "htscore.h"
|
||||
#include "htssafe.h"
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_t_StatsBuffer
|
||||
#define HTS_DEF_FWSTRUCT_t_StatsBuffer
|
||||
typedef struct t_StatsBuffer t_StatsBuffer;
|
||||
#endif
|
||||
struct t_StatsBuffer {
|
||||
char name[1024];
|
||||
char file[1024];
|
||||
char state[256];
|
||||
char BIGSTK url_sav[HTS_URLMAXSIZE * 2]; // pour cancel
|
||||
char BIGSTK url_adr[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK url_fil[HTS_URLMAXSIZE * 2];
|
||||
LLint size;
|
||||
LLint sizetot;
|
||||
int offset;
|
||||
//
|
||||
int back;
|
||||
//
|
||||
int actived; // pour disabled
|
||||
};
|
||||
#include "htsstats.h"
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_t_InpInfo
|
||||
#define HTS_DEF_FWSTRUCT_t_InpInfo
|
||||
|
||||
@@ -100,6 +100,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ClCompile Include="httrack.c" />
|
||||
<ClCompile Include="htsbacktrace.c" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Pulls in libhttrack.lib and builds it first. -->
|
||||
|
||||
@@ -59,9 +59,10 @@
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<!-- LIBHTTRACK_EXPORTS turns HTSEXT_API into __declspec(dllexport); ZLIB_DLL
|
||||
imports zlib from its DLL. HTS_USEBROTLI/HTS_USEZSTD enable the br and
|
||||
zstd content codings. Windows 7 floor, matching WinHTTrack. -->
|
||||
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;HTS_USEBROTLI=1;HTS_USEZSTD=1;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
imports zlib from its DLL, ZLIB_CONST makes its next_in const as the
|
||||
autotools build does. HTS_USEBROTLI/HTS_USEZSTD enable the br and zstd
|
||||
content codings. Windows 7 floor, matching WinHTTrack. -->
|
||||
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;ZLIB_CONST;HTS_USEBROTLI=1;HTS_USEZSTD=1;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>$(MSBuildThisFileDirectory);$(MSBuildThisFileDirectory)coucal;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
@@ -121,6 +122,7 @@
|
||||
<ClCompile Include="htshelp.c" />
|
||||
<ClCompile Include="htsindex.c" />
|
||||
<ClCompile Include="htslib.c" />
|
||||
<ClCompile Include="htscmdline.c" />
|
||||
<ClCompile Include="htsurlport.c" />
|
||||
<ClCompile Include="htsmd5.c" />
|
||||
<ClCompile Include="htsmodules.c" />
|
||||
@@ -129,6 +131,7 @@
|
||||
<ClCompile Include="htsproxy.c" />
|
||||
<ClCompile Include="htsrobots.c" />
|
||||
<ClCompile Include="htsselftest.c" />
|
||||
<ClCompile Include="htssinglefile.c" />
|
||||
<ClCompile Include="htssniff.c" />
|
||||
<ClCompile Include="htsthread.c" />
|
||||
<ClCompile Include="htstools.c" />
|
||||
|
||||
@@ -688,7 +688,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
|
||||
PT_Element elt = PT_ElementNew();
|
||||
|
||||
elt->statuscode = 405;
|
||||
strcpy(elt->msg, "Method Not Allowed");
|
||||
strcpybuff(elt->msg, "Method Not Allowed");
|
||||
return elt;
|
||||
}
|
||||
|
||||
@@ -771,7 +771,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
|
||||
PT_ReadIndex(indexes, StringBuff(itemUrl) + 1, FETCH_HEADERS);
|
||||
if (file != NULL && file->statuscode == HTTP_OK) {
|
||||
size = file->size;
|
||||
if (file->lastmodified) {
|
||||
if (file->lastmodified[0] != '\0') {
|
||||
timestamp = get_time_rfc822(file->lastmodified);
|
||||
}
|
||||
if (timestamp == (time_t) 0) {
|
||||
@@ -785,7 +785,7 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
|
||||
}
|
||||
timestamp = timestampRep;
|
||||
}
|
||||
if (file->contenttype) {
|
||||
if (file->contenttype[0] != '\0') {
|
||||
mimeType = file->contenttype;
|
||||
}
|
||||
}
|
||||
@@ -820,9 +820,9 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
|
||||
elt->size = StringLength(response);
|
||||
elt->adr = StringAcquire(&response);
|
||||
elt->statuscode = 207; /* Multi-Status */
|
||||
strcpy(elt->charset, "utf-8");
|
||||
strcpy(elt->contenttype, "text/xml");
|
||||
strcpy(elt->msg, "Multi-Status");
|
||||
strcpybuff(elt->charset, "utf-8");
|
||||
strcpybuff(elt->contenttype, "text/xml");
|
||||
strcpybuff(elt->msg, "Multi-Status");
|
||||
StringFree(response);
|
||||
|
||||
fprintf(stderr, "RESPONSE:\n%s\n", elt->adr);
|
||||
@@ -877,9 +877,9 @@ static PT_Element proxytrack_process_HTTP_List(PT_Indexes indexes,
|
||||
elt->size = StringLength(html);
|
||||
elt->adr = StringAcquire(&html);
|
||||
elt->statuscode = HTTP_OK;
|
||||
strcpy(elt->charset, "iso-8859-1");
|
||||
strcpy(elt->contenttype, "text/html");
|
||||
strcpy(elt->msg, "OK");
|
||||
strcpybuff(elt->charset, "iso-8859-1");
|
||||
strcpybuff(elt->contenttype, "text/html");
|
||||
strcpybuff(elt->msg, "OK");
|
||||
StringFree(html);
|
||||
return elt;
|
||||
}
|
||||
|
||||
@@ -368,7 +368,11 @@ HTS_UNUSED static struct tm PT_GetTime(time_t t) {
|
||||
if (tm != NULL)
|
||||
return *tm;
|
||||
else {
|
||||
/* an all-zero tm has tm_mday == 0, which the ARC date field prints as a
|
||||
day of "00"; the epoch is the conventional "date unknown" */
|
||||
memset(&tmbuf, 0, sizeof(tmbuf));
|
||||
tmbuf.tm_year = 70;
|
||||
tmbuf.tm_mday = 1;
|
||||
return tmbuf;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,9 +405,9 @@ PT_Element PT_Index_HTML_BuildRootInfo(PT_Indexes indexes) {
|
||||
elt->size = StringLength(html);
|
||||
elt->adr = StringAcquire(&html);
|
||||
elt->statuscode = HTTP_OK;
|
||||
strcpy(elt->charset, "iso-8859-1");
|
||||
strcpy(elt->contenttype, "text/html");
|
||||
strcpy(elt->msg, "OK");
|
||||
strcpybuff(elt->charset, "iso-8859-1");
|
||||
strcpybuff(elt->contenttype, "text/html");
|
||||
strcpybuff(elt->msg, "OK");
|
||||
StringFree(html);
|
||||
return elt;
|
||||
}
|
||||
@@ -579,9 +579,10 @@ PT_Index PT_LoadCache(const char *filename) {
|
||||
if (chain != NULL) {
|
||||
const char *scheme = link_has_authority(chain->name) ? "" : "http://";
|
||||
|
||||
snprintf(index->slots.common.startUrl,
|
||||
sizeof(index->slots.common.startUrl), "%s%s", scheme,
|
||||
(const char *) chain->name);
|
||||
/* dropped rather than truncated: empty already reads as "unset" */
|
||||
if (!sprintfbuff(index->slots.common.startUrl, "%s%s", scheme,
|
||||
(const char *) chain->name))
|
||||
index->slots.common.startUrl[0] = '\0';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -827,6 +828,10 @@ PT_Element PT_ElementNew(void) {
|
||||
return r;
|
||||
}
|
||||
|
||||
/* ProxyTrack's htsblk_failf(): a clipped, diagnostic-only failure reason. */
|
||||
#define PT_Element_failf(R, ...) \
|
||||
slprintfbuff_clip((R)->msg, sizeof((R)->msg), __VA_ARGS__)
|
||||
|
||||
PT_Element PT_ReadCache(PT_Index index, const char *url, int flags) {
|
||||
if (index != NULL && SAFE_INDEX(index)) {
|
||||
return _IndexFuncts[index->type].PT_ReadCache(index, url, flags);
|
||||
@@ -865,12 +870,15 @@ static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags)
|
||||
sprintf(headers + headersSize, "%s: "LLintP"\r\n", field, (LLint)(value)); \
|
||||
(headersSize) += (int) strlen(headers + headersSize); \
|
||||
} while(0)
|
||||
#define ZIP_READFIELD_STRING(line, value, refline, refvalue) do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
strcpy(refvalue, value); \
|
||||
line[0] = '\0'; \
|
||||
} \
|
||||
} while(0)
|
||||
/* refvalue_size is mandatory: the cache line is bounded only by the line
|
||||
buffer, not by the destination. */
|
||||
#define ZIP_READFIELD_STRING(line, value, refline, refvalue, refvalue_size) \
|
||||
do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
(void) strclipbuff(refvalue, refvalue_size, value); \
|
||||
line[0] = '\0'; \
|
||||
} \
|
||||
} while (0)
|
||||
#define ZIP_READFIELD_INT(line, value, refline, refvalue) do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
int intval = 0; \
|
||||
@@ -972,9 +980,12 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
|
||||
const char *scheme =
|
||||
link_has_authority(filenameIndex) ? "" : "http://";
|
||||
|
||||
firstSeen = 1;
|
||||
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
|
||||
scheme, filenameIndex);
|
||||
/* dropped rather than truncated; try the next entry */
|
||||
if (sprintfbuff(index->startUrl, "%s%s", scheme,
|
||||
filenameIndex))
|
||||
firstSeen = 1;
|
||||
else
|
||||
index->startUrl[0] = '\0';
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1021,7 +1032,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
previous_save[0] = previous_save_[0] = '\0';
|
||||
memset(r, 0, sizeof(_PT_Element));
|
||||
r->location = location_default;
|
||||
strcpy(r->location, "");
|
||||
r->location[0] = '\0';
|
||||
if (strncmp(url, "http://", 7) == 0)
|
||||
url += 7;
|
||||
hash_pos_return = coucal_read(index->hash, url, &hash_pos);
|
||||
@@ -1070,16 +1081,23 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
value++;
|
||||
ZIP_READFIELD_INT(line, value, "X-In-Cache", dataincache);
|
||||
ZIP_READFIELD_INT(line, value, "X-Statuscode", r->statuscode);
|
||||
ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg); // msg
|
||||
ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg,
|
||||
sizeof(r->msg));
|
||||
ZIP_READFIELD_INT(line, value, "X-Size", r->size); // size
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype
|
||||
ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset); // contenttype
|
||||
ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified
|
||||
ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag
|
||||
ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype,
|
||||
sizeof(r->contenttype));
|
||||
ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset,
|
||||
sizeof(r->charset));
|
||||
ZIP_READFIELD_STRING(line, value, "Last-Modified",
|
||||
r->lastmodified, sizeof(r->lastmodified));
|
||||
ZIP_READFIELD_STRING(line, value, "Etag", r->etag,
|
||||
sizeof(r->etag));
|
||||
ZIP_READFIELD_STRING(line, value, "Location", r->location,
|
||||
sizeof(location_default));
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition",
|
||||
r->cdispo); // Content-disposition
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename
|
||||
r->cdispo, sizeof(r->cdispo));
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
|
||||
sizeof(previous_save_));
|
||||
if (line[0] != '\0') {
|
||||
int len = r->headers ? ((int) strlen(r->headers)) : 0;
|
||||
int nlen =
|
||||
@@ -1099,7 +1117,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
int pathLen = (int) strlen(index->path);
|
||||
|
||||
if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format
|
||||
strcpy(previous_save, previous_save_);
|
||||
strcpybuff(previous_save, previous_save_);
|
||||
}
|
||||
// relative ? (hack)
|
||||
else if (index->safeCache || (previous_save_[0] != '/' // /home/foo/bar.gif
|
||||
@@ -1135,8 +1153,9 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s",
|
||||
index->path, previous_save_ + index->fixedPath);
|
||||
} else {
|
||||
snprintf(r->msg, sizeof(r->msg), "Bogus fixePath prefix for %s (prefixLen=%d)",
|
||||
previous_save_, (int) index->fixedPath);
|
||||
PT_Element_failf(
|
||||
r, "Bogus fixePath prefix for %s (prefixLen=%d)",
|
||||
previous_save_, (int) index->fixedPath);
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
@@ -1155,7 +1174,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
// Peut-on stocker le fichier directement sur disque?
|
||||
if (ok) {
|
||||
if (r->msg[0] == '\0') {
|
||||
strcpy(r->msg, "Cache Read Error : Unexpected error");
|
||||
strcpybuff(r->msg, "Cache Read Error : Unexpected error");
|
||||
}
|
||||
} else { // lire en mémoire
|
||||
|
||||
@@ -1173,24 +1192,27 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
int last_errno = errno;
|
||||
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
sprintf(r->msg, "Read error in cache disk data: %s",
|
||||
strerror(last_errno));
|
||||
PT_Element_failf(r,
|
||||
"Read error in cache disk data: %s",
|
||||
strerror(last_errno));
|
||||
}
|
||||
r->adr[r->size] = '\0';
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg,
|
||||
"Read error (memory exhausted) from cache");
|
||||
strcpybuff(r->msg,
|
||||
"Read error (memory exhausted) from cache");
|
||||
}
|
||||
fclose(fp);
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
snprintf(r->msg, sizeof(r->msg), "Read error (can't open '%s') from cache",
|
||||
file_convert(catbuff, sizeof(catbuff), previous_save));
|
||||
PT_Element_failf(
|
||||
r, "Read error (can't open '%s') from cache",
|
||||
file_convert(catbuff, sizeof(catbuff),
|
||||
previous_save));
|
||||
}
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cached file name is invalid");
|
||||
strcpybuff(r->msg, "Cached file name is invalid");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1202,12 +1224,12 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
free(r->adr);
|
||||
r->adr = NULL;
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Read Data");
|
||||
strcpybuff(r->msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
*(r->adr + r->size) = '\0';
|
||||
} else { // erreur
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Memory Error");
|
||||
strcpybuff(r->msg, "Cache Memory Error");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1215,21 +1237,21 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
} // si save==null, ne rien charger (juste en tête)
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Read Header Data");
|
||||
strcpybuff(r->msg, "Cache Read Error : Read Header Data");
|
||||
}
|
||||
unzCloseCurrentFile(index->zFile);
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Open File");
|
||||
strcpybuff(r->msg, "Cache Read Error : Open File");
|
||||
}
|
||||
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Bad Offset");
|
||||
strcpybuff(r->msg, "Cache Read Error : Bad Offset");
|
||||
}
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "File Cache Entry Not Found");
|
||||
strcpybuff(r->msg, "File Cache Entry Not Found");
|
||||
}
|
||||
if (r->location[0] != '\0') {
|
||||
r->location = strdup(r->location);
|
||||
@@ -1522,7 +1544,7 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
cache->version = (int) (firstline[8] - '0'); // cache 1.x
|
||||
if (cache->version <= 5) {
|
||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
||||
strcpy(cache->lastmodified, firstline);
|
||||
strcpybuff(cache->lastmodified, firstline);
|
||||
} else {
|
||||
fclose(cache->dat);
|
||||
cache->dat = NULL;
|
||||
@@ -1539,7 +1561,7 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
} else { // Vieille version du cache
|
||||
/* */
|
||||
cache->version = 0; // cache 1.0
|
||||
strcpy(cache->lastmodified, firstline);
|
||||
strcpybuff(cache->lastmodified, firstline);
|
||||
}
|
||||
|
||||
/* Create hash table for the cache (MUCH FASTER!) */
|
||||
@@ -1555,7 +1577,14 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
a++;
|
||||
/* read "host/file" */
|
||||
a += binput(a, line, HTS_URLMAXSIZE);
|
||||
a += binput(a, line + strlen(line), HTS_URLMAXSIZE);
|
||||
{
|
||||
/* binput writes its NUL at s[max], so the second field must be
|
||||
bounded by what is left of line[], not by the same constant
|
||||
*/
|
||||
const size_t used = strlen(line);
|
||||
|
||||
a += binput(a, line + used, (int) (sizeof(line) - used - 1));
|
||||
}
|
||||
/* read position */
|
||||
a += binput(a, linepos, 200);
|
||||
sscanf(linepos, "%d", &pos);
|
||||
@@ -1570,9 +1599,11 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
const char *scheme =
|
||||
link_has_authority(line) ? "" : "http://";
|
||||
|
||||
firstSeen = 1;
|
||||
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
|
||||
scheme, line);
|
||||
/* dropped rather than truncated; try the next entry */
|
||||
if (sprintfbuff(index->startUrl, "%s%s", scheme, line))
|
||||
firstSeen = 1;
|
||||
else
|
||||
index->startUrl[0] = '\0';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1650,7 +1681,7 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
previous_save[0] = previous_save_[0] = '\0';
|
||||
memset(r, 0, sizeof(_PT_Element));
|
||||
r->location = location_default;
|
||||
strcpy(r->location, "");
|
||||
r->location[0] = '\0';
|
||||
if (strncmp(url, "http://", 7) == 0)
|
||||
url += 7;
|
||||
hash_pos_return = coucal_read(cache->hash, url, &hash_pos);
|
||||
@@ -1757,7 +1788,7 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
int pathLen = (int) strlen(index->path);
|
||||
|
||||
if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format
|
||||
strcpy(previous_save, previous_save_);
|
||||
strcpybuff(previous_save, previous_save_);
|
||||
}
|
||||
// relative ? (hack)
|
||||
else if (index->safeCache || (previous_save_[0] != '/' // /home/foo/bar.gif
|
||||
@@ -1793,8 +1824,9 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s",
|
||||
index->path, previous_save_ + index->fixedPath);
|
||||
} else {
|
||||
snprintf(r->msg, sizeof(r->msg), "Bogus fixePath prefix for %s (prefixLen=%d)",
|
||||
previous_save_, (int) index->fixedPath);
|
||||
PT_Element_failf(r,
|
||||
"Bogus fixePath prefix for %s (prefixLen=%d)",
|
||||
previous_save_, (int) index->fixedPath);
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
@@ -1820,17 +1852,18 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
if (r->adr != NULL) {
|
||||
if (r->size > 0 && fread(r->adr, 1, r->size, fp) != r->size) {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Read error in cache disk data");
|
||||
strcpybuff(r->msg, "Read error in cache disk data");
|
||||
}
|
||||
r->adr[r->size] = '\0';
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Read error (memory exhausted) from cache");
|
||||
strcpybuff(r->msg,
|
||||
"Read error (memory exhausted) from cache");
|
||||
}
|
||||
fclose(fp);
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Previous cache file not found (2)");
|
||||
strcpybuff(r->msg, "Previous cache file not found (2)");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1842,30 +1875,30 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
free(r->adr);
|
||||
r->adr = NULL;
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Read Data");
|
||||
strcpybuff(r->msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
r->adr[r->size] = '\0';
|
||||
} else { // erreur
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Memory Error");
|
||||
strcpybuff(r->msg, "Cache Memory Error");
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Bad Data");
|
||||
strcpybuff(r->msg, "Cache Read Error : Bad Data");
|
||||
}
|
||||
} else { // erreur
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Read Header");
|
||||
strcpybuff(r->msg, "Cache Read Error : Read Header");
|
||||
}
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Seek Failed");
|
||||
strcpybuff(r->msg, "Cache Read Error : Seek Failed");
|
||||
}
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "File Cache Entry Not Found");
|
||||
strcpybuff(r->msg, "File Cache Entry Not Found");
|
||||
}
|
||||
if (r->location[0] != '\0') {
|
||||
r->location = strdup(r->location);
|
||||
@@ -2128,12 +2161,14 @@ int PT_LoadCache__Arc(PT_Index index_, const char *filename) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define HTTP_READFIELD_STRING(line, value, refline, refvalue) do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
strcpy(refvalue, value); \
|
||||
line[0] = '\0'; \
|
||||
} \
|
||||
} while(0)
|
||||
/* Same contract as ZIP_READFIELD_STRING, for the ARC reader's header lines. */
|
||||
#define HTTP_READFIELD_STRING(line, value, refline, refvalue, refvalue_size) \
|
||||
do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
(void) strclipbuff(refvalue, refvalue_size, value); \
|
||||
line[0] = '\0'; \
|
||||
} \
|
||||
} while (0)
|
||||
#define HTTP_READFIELD_INT(line, value, refline, refvalue) do { \
|
||||
if (line[0] != '\0' && strfield2(line, refline)) { \
|
||||
int intval = 0; \
|
||||
@@ -2169,7 +2204,7 @@ static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
|
||||
location_default[0] = '\0';
|
||||
memset(r, 0, sizeof(_PT_Element));
|
||||
r->location = location_default;
|
||||
strcpy(r->location, "");
|
||||
r->location[0] = '\0';
|
||||
if (strncmp(url, "http://", 7) == 0)
|
||||
url += 7;
|
||||
hash_pos_return = coucal_read(index->hash, url, &hash_pos);
|
||||
@@ -2191,7 +2226,7 @@ static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
|
||||
}
|
||||
if ((pos = getArcField(index->line, 2)) != NULL) {
|
||||
r->msg[0] = '\0';
|
||||
strncat(r->msg, pos, sizeof(pos) - 1);
|
||||
strncatbuff(r->msg, pos, sizeof(r->msg) - 1);
|
||||
}
|
||||
while(linput(index->file, index->line, sizeof(index->line) - 1)
|
||||
&& index->line[0] != '\0') {
|
||||
@@ -2202,11 +2237,16 @@ static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
|
||||
*value = '\0';
|
||||
for(value++; *value == ' ' || *value == '\t'; value++) ;
|
||||
HTTP_READFIELD_INT(line, value, "Content-Length", r->size); // size
|
||||
HTTP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype
|
||||
HTTP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified
|
||||
HTTP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag
|
||||
HTTP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved
|
||||
HTTP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition
|
||||
HTTP_READFIELD_STRING(line, value, "Content-Type", r->contenttype,
|
||||
sizeof(r->contenttype));
|
||||
HTTP_READFIELD_STRING(line, value, "Last-Modified",
|
||||
r->lastmodified, sizeof(r->lastmodified));
|
||||
HTTP_READFIELD_STRING(line, value, "Etag", r->etag,
|
||||
sizeof(r->etag));
|
||||
HTTP_READFIELD_STRING(line, value, "Location", r->location,
|
||||
sizeof(location_default));
|
||||
HTTP_READFIELD_STRING(line, value, "Content-Disposition",
|
||||
r->cdispo, sizeof(r->cdispo));
|
||||
if (line[0] != '\0') {
|
||||
int len = r->headers ? ((int) strlen(r->headers)) : 0;
|
||||
int nlen =
|
||||
@@ -2246,7 +2286,7 @@ static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
|
||||
fetchSize = dataLength - metaSize;
|
||||
} else if (fetchSize > dataLength - metaSize) {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Truncated Data");
|
||||
strcpybuff(r->msg, "Cache Read Error : Truncated Data");
|
||||
}
|
||||
r->size = 0;
|
||||
if (r->statuscode != STATUSCODE_INVALID) {
|
||||
@@ -2259,12 +2299,13 @@ static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
|
||||
int last_errno = errno;
|
||||
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
sprintf(r->msg, "Read error in cache disk data: %s",
|
||||
strerror(last_errno));
|
||||
PT_Element_failf(r, "Read error in cache disk data: %s",
|
||||
strerror(last_errno));
|
||||
}
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Read error (memory exhausted) from cache");
|
||||
strcpybuff(r->msg,
|
||||
"Read error (memory exhausted) from cache");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2272,21 +2313,21 @@ static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
|
||||
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Read Header Error");
|
||||
strcpybuff(r->msg, "Cache Read Error : Read Header Error");
|
||||
}
|
||||
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Read Header Error");
|
||||
strcpybuff(r->msg, "Cache Read Error : Read Header Error");
|
||||
}
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Read Error : Seek Error");
|
||||
strcpybuff(r->msg, "Cache Read Error : Seek Error");
|
||||
}
|
||||
|
||||
} else {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "File Cache Entry Not Found");
|
||||
strcpybuff(r->msg, "File Cache Entry Not Found");
|
||||
}
|
||||
if (r->location[0] != '\0') {
|
||||
r->location = strdup(r->location);
|
||||
@@ -2335,8 +2376,18 @@ static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element)
|
||||
PT_SaveCache__Arc_t *st = (PT_SaveCache__Arc_t *) arg;
|
||||
FILE *const fp = st->fp;
|
||||
struct tm *tm = convert_time_rfc822(&st->buff, element->lastmodified);
|
||||
struct tm unknown_date;
|
||||
int size_headers;
|
||||
|
||||
/* a cached entry with no parseable Last-Modified must not take the writer
|
||||
down; the epoch is the conventional "date unknown" */
|
||||
if (tm == NULL) {
|
||||
memset(&unknown_date, 0, sizeof(unknown_date));
|
||||
unknown_date.tm_year = 70;
|
||||
unknown_date.tm_mday = 1;
|
||||
tm = &unknown_date;
|
||||
}
|
||||
|
||||
sprintf(st->headers,
|
||||
"HTTP/1.0 %d %s"
|
||||
"\r\n"
|
||||
@@ -2372,7 +2423,7 @@ static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element)
|
||||
if (element->adr != NULL) {
|
||||
domd5mem(element->adr, element->size, st->md5, 1);
|
||||
} else {
|
||||
strcpy(st->md5, "-");
|
||||
strcpybuff(st->md5, "-");
|
||||
}
|
||||
fprintf(fp,
|
||||
/* nl */
|
||||
|
||||
@@ -97,6 +97,7 @@
|
||||
<ItemGroup>
|
||||
<ClCompile Include="htsserver.c" />
|
||||
<ClCompile Include="htsweb.c" />
|
||||
<ClCompile Include="htscmdline.c" />
|
||||
<ClCompile Include="htsurlport.c" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
8
tests/01_engine-cmdline-split.test
Executable file
8
tests/01_engine-cmdline-split.test
Executable file
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# webhttrack posts its command line as one string: the argv split must grow past
|
||||
# 1024 arguments and keep a quote inside a value out of the option parser.
|
||||
httrack -O /dev/null -#test=cmdline-split run | grep -q "cmdline-split self-test OK"
|
||||
25
tests/01_engine-growsize.test
Normal file
25
tests/01_engine-growsize.test
Normal file
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Buffer capacity for a 64-bit file size ('httrack -#test=growsize'): a -%S list
|
||||
# file past 4GB must size its buffer exactly or be refused, never wrap.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
out=$(httrack -#test=growsize)
|
||||
echo "$out"
|
||||
test "$out" == "growsize self-test OK"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_growsize.XXXXXX")
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
echo '<html><body>hi</body></html>' >"$tmp/index.html"
|
||||
printf -- '-*/zzmarker*\n' >"$tmp/rules.txt"
|
||||
|
||||
# the rules file lands in the URL/filter string, echoed back by the banner
|
||||
run=$(httrack -O "$tmp/out" --quiet -n "-%S" "$tmp/rules.txt" \
|
||||
"file://$tmp/index.html" 2>&1) || true
|
||||
printf '%s\n' "$run" | grep -q 'zzmarker' || {
|
||||
echo "FAIL: -%S rules file was not loaded"
|
||||
printf '%s\n' "$run"
|
||||
exit 1
|
||||
}
|
||||
@@ -47,3 +47,18 @@ case "$err" in
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# An array source with no NUL must abort on the source bound rather than run
|
||||
# off the array: the array half of the source-capacity selection.
|
||||
err=$(httrack -#test=strsafe overflow-src "x" 2>&1) || true
|
||||
case "$err" in
|
||||
*"strsafe: NOT aborted"*)
|
||||
echo "unterminated source array was NOT caught" >&2
|
||||
exit 1
|
||||
;;
|
||||
*"size < sizeof_source"*) ;;
|
||||
*)
|
||||
echo "expected htssafe source-bound abort, got: $err" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user