mirror of
https://github.com/xroche/httrack.git
synced 2026-07-12 20:07:43 +03:00
Compare commits
1 Commits
fix-codeql
...
fix-x32-ll
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e327d90c3b |
32
.github/workflows/ci.yml
vendored
32
.github/workflows/ci.yml
vendored
@@ -55,9 +55,7 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -101,8 +99,7 @@ jobs:
|
||||
sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
|
||||
-exec mv {} {}.hidden \;
|
||||
! command -v python3
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -136,15 +133,7 @@ jobs:
|
||||
run: make -j"$(sysctl -n hw.ncpu)"
|
||||
|
||||
- name: Test
|
||||
# bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
|
||||
# it competes with other crawls, flaking its exact file count (the #527
|
||||
# macOS drop). Run everything else in parallel, then bigcrawl alone (its
|
||||
# serial-safe condition). Linux tolerates the full parallel run.
|
||||
run: |
|
||||
jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
|
||||
make check -j"$jobs" TESTS="$rest"
|
||||
make check TESTS=36_local-bigcrawl.test
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -182,9 +171,7 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -239,9 +226,7 @@ jobs:
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -287,8 +272,7 @@ jobs:
|
||||
# 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
|
||||
# skipped here (uninstrumented libz floods MSan with false positives).
|
||||
tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs" TESTS="$tests"
|
||||
make check TESTS="$tests"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -363,9 +347,7 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
|
||||
15
AGENTS.md
15
AGENTS.md
@@ -6,19 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
## Build & test
|
||||
- Fresh clone first: `git submodule update --init src/coucal`
|
||||
- `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
|
||||
automake, libtool), then `bash configure && make -j"$(nproc)" && make check
|
||||
-j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
|
||||
automake's parallel harness and each crawl test binds its own ephemeral-port
|
||||
server, so `-j` never contends and a multi-minute serial run drops to
|
||||
seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
|
||||
automatically; only a test slower than the current longest raises the floor.
|
||||
On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
|
||||
spend much of their wall time asleep (server trickles, httrack self-pacing),
|
||||
so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
|
||||
36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
|
||||
the macOS loopback when it competes with other crawls and flakes its exact
|
||||
file count (Linux tolerates the full parallel run).
|
||||
Or run `sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
automake, libtool), then `bash configure && make && make check`. Or run
|
||||
`sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
|
||||
## Hard invariants
|
||||
- **Generated autotools files are NOT in git.** `configure`, every
|
||||
|
||||
@@ -3512,11 +3512,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
continue;
|
||||
}
|
||||
|
||||
// The server really sent 304 here; the *-hacks below force
|
||||
// NOT_MODIFIED only after confirming the file is complete.
|
||||
const hts_boolean server_sent_304 =
|
||||
(back[i].r.statuscode == HTTP_NOT_MODIFIED);
|
||||
|
||||
/*
|
||||
Solve "false" 416 problems
|
||||
*/
|
||||
@@ -3545,20 +3540,22 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (back[i].r.statuscode == 406) { // 'Not Acceptable'
|
||||
back[i].r.statuscode = HTTP_OK;
|
||||
}
|
||||
// On update, keep the good copy on error (mask as 304); skip
|
||||
// resume paths so a stale-partial 416 still re-fetches.
|
||||
if (HTTP_IS_ERROR(back[i].r.statuscode) &&
|
||||
back[i].is_update && !back[i].testmode &&
|
||||
back[i].range_req_size == 0 && back[i].url_sav[0] &&
|
||||
fexist_utf8(back[i].url_sav)) {
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"Error %d (%s) ignored on update, keeping "
|
||||
"existing copy: %s%s",
|
||||
back[i].r.statuscode, back[i].r.msg,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
back[i].r.statuscode = HTTP_NOT_MODIFIED;
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
// 'do not erase already downloaded file'
|
||||
// on an updated file
|
||||
// with an error : consider a 304 error
|
||||
if (!opt->delete_old) {
|
||||
if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
|
||||
&& !back[i].testmode) {
|
||||
if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"Error ignored %d (%s) because of 'no purge' option for %s%s",
|
||||
back[i].r.statuscode, back[i].r.msg,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
back[i].r.statuscode = HTTP_NOT_MODIFIED;
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Various hacks to limit re-transfers when updating a mirror
|
||||
// Force update if same size detected
|
||||
@@ -3611,7 +3608,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) { // 'OK'
|
||||
if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) { // not HTML
|
||||
if (strnotempty(back[i].url_sav)) { // target found
|
||||
off_t size = fsize_utf8(back[i].url_sav);
|
||||
int size = fsize_utf8(back[i].url_sav); // target size
|
||||
|
||||
if (size >= 0) {
|
||||
if (back[i].r.totalsize == size) { // same size!
|
||||
@@ -3706,21 +3703,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
}
|
||||
|
||||
// Out-of-protocol 304 to a Range resume: the file is still
|
||||
// partial, so drop it and refetch instead of trusting it.
|
||||
if (server_sent_304 && back[i].range_req_size > 0) {
|
||||
url_savename_refname_remove(opt, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
UNLINK(back[i].url_sav);
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
strcpybuff(back[i].r.msg,
|
||||
"Bogus 304 on resume, restarting");
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
}
|
||||
|
||||
/* sinon, continuer */
|
||||
/* if (back[i].r.soc!=INVALID_SOCKET) { // ok récupérer body? */
|
||||
// head: terminé
|
||||
@@ -3839,7 +3821,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
const hts_boolean range_ok =
|
||||
back[i].r.crange > 0 && resume >= 0 &&
|
||||
resume <= (LLint) sz &&
|
||||
back[i].r.crange_end == back[i].r.crange - 1 &&
|
||||
back[i].r.crange_end + 1 == back[i].r.crange &&
|
||||
(back[i].r.totalsize < 0 ||
|
||||
back[i].r.totalsize ==
|
||||
back[i].r.crange_end - resume + 1);
|
||||
@@ -3908,20 +3890,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (fp) {
|
||||
LLint alloc_mem = resume + 1;
|
||||
|
||||
// Bound the in-memory buffer to a 32-bit size (real
|
||||
// in-RAM resources are far smaller); a hostile
|
||||
// Content-Length that would overflow the add or the
|
||||
// (size_t) cast is dropped and refetched instead.
|
||||
if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
|
||||
url_savename_refname_remove(opt, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
UNLINK(back[i].url_sav);
|
||||
alloc_mem = -1;
|
||||
} else if (back[i].r.totalsize >= 0)
|
||||
if (back[i].r.totalsize >= 0)
|
||||
alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
|
||||
if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
|
||||
(back[i].r.adr =
|
||||
(char *) malloct((size_t) alloc_mem))) {
|
||||
if (deleteaddr(&back[i].r)
|
||||
&& (back[i].r.adr =
|
||||
(char *) malloct((size_t) alloc_mem))) {
|
||||
back[i].r.size = resume;
|
||||
if (back[i].r.totalsize >= 0)
|
||||
back[i].r.totalsize += resume; // -> full size
|
||||
@@ -4207,13 +4180,10 @@ static int back_maxtime_grace(const int maxtime) {
|
||||
No floor (unlike the time grace): a size overrun should abort promptly. */
|
||||
static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
|
||||
|
||||
/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
|
||||
-M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
|
||||
which undercounts redirect/error-heavy crawls (#520). */
|
||||
/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
|
||||
static hts_mirror_limit back_mirror_limit(httrackp *opt) {
|
||||
if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
|
||||
HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
|
||||
back_maxsize_grace(opt->maxsite))
|
||||
if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
|
||||
HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
|
||||
return HTS_MIRROR_LIMIT_SIZE;
|
||||
if (opt->maxtime > 0) {
|
||||
const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
|
||||
@@ -4227,7 +4197,7 @@ static hts_mirror_limit back_mirror_limit(httrackp *opt) {
|
||||
|
||||
int back_checkmirror(httrackp *opt) {
|
||||
/* request a smooth stop the first time each cap is reached */
|
||||
if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
|
||||
if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
|
||||
!opt->state.stop) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"More than " LLintP
|
||||
|
||||
@@ -56,10 +56,6 @@ Please visit our Website: http://www.httrack.com
|
||||
// Platform detection (sizes, feature macros)
|
||||
#include "htsconfig.h"
|
||||
|
||||
// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
|
||||
#include <stdint.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
// WIN32 types
|
||||
#ifdef _WIN32
|
||||
#ifndef SIZEOF_LONG
|
||||
@@ -115,6 +111,29 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_DO_NOT_USE_UID
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef SIZEOF_LONG_LONG
|
||||
#if SIZEOF_LONG_LONG == 8
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef __sun
|
||||
#define HTS_LONGLONG 0
|
||||
#endif
|
||||
#ifdef __osf__
|
||||
#define HTS_LONGLONG 0
|
||||
#endif
|
||||
#ifdef __linux
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#ifdef _WIN32
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef DLLIB
|
||||
#define HTS_DLOPEN 1
|
||||
#else
|
||||
@@ -307,11 +326,66 @@ typedef int hts_tristate;
|
||||
#define HTS_DEPRECATED(msg)
|
||||
#endif
|
||||
|
||||
/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
|
||||
typedef int64_t LLint;
|
||||
typedef int64_t TStamp;
|
||||
/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
|
||||
#define LLintP "%" PRId64
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef HTS_NO_64_BIT
|
||||
#define HTS_LONGLONG 0
|
||||
#else
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* Wide integer types, chosen per platform.
|
||||
LLint: signed 64-bit counter for byte counts and large sizes (falls back to
|
||||
plain int where 64-bit is unavailable).
|
||||
TStamp: timestamp/duration in the same width (a double in the no-64-bit
|
||||
fallback).
|
||||
LLintP: the printf conversion for an LLint. */
|
||||
#if HTS_LONGLONG
|
||||
#ifdef LLINT_FORMAT
|
||||
typedef LLINT_TYPE LLint;
|
||||
|
||||
typedef LLINT_TYPE TStamp;
|
||||
|
||||
#define LLintP LLINT_FORMAT
|
||||
#else
|
||||
|
||||
#ifdef _WIN32
|
||||
typedef __int64 LLint;
|
||||
|
||||
typedef __int64 TStamp;
|
||||
|
||||
#define LLintP "%I64d"
|
||||
/* x32/ILP32 sets __x86_64__ but long is 32-bit; exclude it so LLint stays
|
||||
* 64-bit. */
|
||||
#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) || \
|
||||
defined(__64BIT__)) && \
|
||||
!defined(__ILP32__)
|
||||
|
||||
typedef long int LLint;
|
||||
|
||||
typedef long int TStamp;
|
||||
|
||||
#define LLintP "%ld"
|
||||
#else
|
||||
typedef long long int LLint;
|
||||
|
||||
typedef long long int TStamp;
|
||||
|
||||
#define LLintP "%lld"
|
||||
#endif
|
||||
|
||||
#endif /* HTS_LONGLONG */
|
||||
|
||||
/* Claiming long-long support must yield a real 64-bit LLint (x32 regressed:
|
||||
__x86_64__ set but long is 32-bit). Compile-time trip, portable to C90. */
|
||||
typedef char hts_assert_llint_is_64bit[sizeof(LLint) == 8 ? 1 : -1];
|
||||
|
||||
#else
|
||||
typedef int LLint;
|
||||
|
||||
#define LLintP "%d"
|
||||
typedef double TStamp;
|
||||
#endif
|
||||
|
||||
/* Integer type for file offsets/sizes passed to the C library. Widens to
|
||||
LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
|
||||
|
||||
30
src/htslib.c
30
src/htslib.c
@@ -1586,7 +1586,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
a = strchr(rcvd + p, '/');
|
||||
if (a != NULL) {
|
||||
a++;
|
||||
if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
|
||||
if (sscanf(a, LLintP, &retour->crange) == 1) {
|
||||
retour->crange_start = 0;
|
||||
retour->crange_end = retour->crange - 1;
|
||||
} else {
|
||||
@@ -1594,12 +1594,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
}
|
||||
}
|
||||
}
|
||||
// A valid Content-Range has no negative field; reject hostile values so
|
||||
// the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
|
||||
if (retour->crange_start < 0 || retour->crange_end < 0 ||
|
||||
retour->crange < 0) {
|
||||
retour->crange_start = retour->crange_end = retour->crange = 0;
|
||||
}
|
||||
}
|
||||
} else if ((p = strfield(rcvd, "Connection:")) != 0) {
|
||||
char *a = rcvd + p;
|
||||
@@ -3044,11 +3038,12 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
|
||||
/* See http://physics.nist.gov/cuu/Units/binary.html */
|
||||
#define ToLLint(a) ((LLint)(a))
|
||||
#define ToLLintKiB (ToLLint(1024))
|
||||
#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintPiB \
|
||||
(ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
|
||||
#ifdef HTS_LONGLONG
|
||||
#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#endif
|
||||
HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
if (n < ToLLintKiB) {
|
||||
sprintf(strc->buff1, "%d", (int) (LLint) n);
|
||||
@@ -3057,7 +3052,9 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
|
||||
(int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
|
||||
strcpybuff(strc->buff2, "KiB");
|
||||
} else if (n < ToLLintGiB) {
|
||||
}
|
||||
#ifdef HTS_LONGLONG
|
||||
else if (n < ToLLintGiB) {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
|
||||
(int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
|
||||
strcpybuff(strc->buff2, "MiB");
|
||||
@@ -3074,6 +3071,13 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
(int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
|
||||
strcpybuff(strc->buff2, "PiB");
|
||||
}
|
||||
#else
|
||||
else {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
|
||||
(int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
|
||||
strcpybuff(strc->buff2, "MiB");
|
||||
}
|
||||
#endif
|
||||
strc->buffadr[0] = strc->buff1;
|
||||
strc->buffadr[1] = strc->buff2;
|
||||
return strc->buffadr;
|
||||
|
||||
@@ -1310,29 +1310,6 @@ static int st_headerlong(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Parse a Content-Range header and print the sanitized triple. A hostile value
|
||||
(negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
|
||||
static int st_crange(httrackp *opt, int argc, char **argv) {
|
||||
int i;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "crange: needs at least one raw Content-Range line\n");
|
||||
return 1;
|
||||
}
|
||||
for (i = 0; i < argc; i++) {
|
||||
htsblk r;
|
||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
||||
|
||||
memset(&r, 0, sizeof(r));
|
||||
strcpybuff(line, argv[i]);
|
||||
treathead(NULL, "www.example.com", "/", &r, line);
|
||||
printf("crange_start=" LLintP " crange_end=" LLintP " crange=" LLintP "\n",
|
||||
(LLint) r.crange_start, (LLint) r.crange_end, (LLint) r.crange);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
|
||||
static size_t st_decode_body(const char *arg, char *buf, size_t size) {
|
||||
size_t n = 0;
|
||||
@@ -2519,8 +2496,6 @@ static const struct selftest_entry {
|
||||
{"headerlong", "[header-name:]",
|
||||
"over-long header value must not overflow the parse scratch",
|
||||
st_headerlong},
|
||||
{"crange", "<raw-content-range-line> ...",
|
||||
"Content-Range parse integer safety", st_crange},
|
||||
{"savename", "<fil> <content-type> [key=value ...]",
|
||||
"local save-name for a URL", st_savename},
|
||||
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Content-Range parse (treathead via -#test=crange). Hostile values must clamp
|
||||
# to 0, and the crange +/- 1 arithmetic must not sign-overflow (UBSan aborts on
|
||||
# the pre-fix binary at the INT64_MIN cases).
|
||||
|
||||
cr() {
|
||||
local want="$1"
|
||||
shift
|
||||
out="$(httrack -O /dev/null -#test=crange "$@")"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: $* -> '$out' (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Normal range and the '*/N' fallback both parse through unchanged.
|
||||
cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes 0-70870/70871'
|
||||
cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes */70871'
|
||||
|
||||
# INT64_MIN total in the fallback would make crange-1 overflow: clamp to 0.
|
||||
cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes */-9223372036854775808'
|
||||
|
||||
# Any negative field in the 3-field form clamps the whole triple to 0 (each
|
||||
# field independently: a leading negative start/end alone still zeroes all).
|
||||
cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -1--1/-1'
|
||||
cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -5-10/20'
|
||||
|
||||
# INT64_MAX is non-negative: it passes through, the parse must not mangle it.
|
||||
cr 'crange_start=0 crange_end=9223372036854775807 crange=9223372036854775807' \
|
||||
'Content-Range: bytes 0-9223372036854775807/9223372036854775807'
|
||||
@@ -66,12 +66,9 @@ test -n "$port" || {
|
||||
}
|
||||
|
||||
out="$tmpdir/crawl"
|
||||
# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
|
||||
# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
|
||||
# bound the stall without changing what the fallback exercises.
|
||||
HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
|
||||
httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
|
||||
-c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
|
||||
-c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
|
||||
|
||||
log="$out/hts-log.txt"
|
||||
|
||||
@@ -95,12 +92,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# every address dead: the slot exhausts the list, then errors out (the harness
|
||||
# timeout would catch a hang/loop)
|
||||
# every address refused: the slot exhausts the list, then errors out (the
|
||||
# harness timeout would catch a hang/loop; refused connects are instant)
|
||||
out2="$tmpdir/crawl2"
|
||||
HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
|
||||
httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
|
||||
-c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
|
||||
-c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
|
||||
log2="$out2/hts-log.txt"
|
||||
|
||||
grep -q "trying next address" "$log2" || {
|
||||
|
||||
@@ -17,10 +17,7 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
|
||||
printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
|
||||
>"$tmproot/javaclass/Foo.class"
|
||||
|
||||
# --log-found first: a plugin-load failure (wrong soname, shadowing system
|
||||
# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
|
||||
--log-found '\+libhtsjava' \
|
||||
--found 'javaclass/Foo.class' \
|
||||
--found 'javaclass/hello.gif' \
|
||||
httrack 'BASEURL/javaclass/index.html'
|
||||
|
||||
@@ -9,7 +9,7 @@ set -euo pipefail
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --rerun \
|
||||
--errors-content 4 --files 361 \
|
||||
--errors 4 --files 361 \
|
||||
--found 'big/p/95.html' \
|
||||
--found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
|
||||
--found 'big/a/f2-2x.png' \
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
|
||||
# keep its good local copy, not be overwritten by the error body nor purged.
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --rerun \
|
||||
--found 'errmask/keep.dat' \
|
||||
--file-min-bytes 'errmask/keep.dat' 1024 \
|
||||
--file-matches 'errmask/keep.dat' '^KEEP' \
|
||||
--file-not-matches 'errmask/keep.dat' 'error 403' \
|
||||
httrack 'BASEURL/errmask/index.html'
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# -M caps received volume, not saved 200-only bytes (#520): links to large 404
|
||||
# bodies pump received >> saved; the cap must trip though little lands on disk.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# The mirror bound is the load-bearing half: it fires while <100 KB is saved,
|
||||
# so the cap can only have tripped on received volume, not saved bytes (which
|
||||
# would exceed the cap and fetch all 16 links).
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--log-found 'More than 500000 bytes have been transferred.. giving up' \
|
||||
--max-mirror-bytes 100000 \
|
||||
httrack 'BASEURL/maxrecv/index.html' -M500000 -c4
|
||||
@@ -1,118 +0,0 @@
|
||||
#!/bin/bash
|
||||
# C7: a server 304 answering a resume's Range request is out of protocol; httrack
|
||||
# must drop the partial and refetch, not finalize it as complete at the partial
|
||||
# byte count. Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets a bogus
|
||||
# 304, and must recover the whole file.
|
||||
set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
counter="${tmpdir}/reqcount"
|
||||
mark="${tmpdir}/got304"
|
||||
RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
|
||||
refdir="${out}/hts-cache/ref"
|
||||
|
||||
# --- pass 1: crawl, interrupt once the blob download is underway -------------
|
||||
printf '[pass 1: interrupt mid-download] ..\t'
|
||||
httrack "${common[@]}" "${base}/resume304/index.html" >"${tmpdir}/log1" 2>&1 &
|
||||
crawlpid=$!
|
||||
for _ in $(seq 1 300); do
|
||||
test -s "$counter" && break
|
||||
kill -0 "$crawlpid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
sleep 0.5
|
||||
kill -TERM "$crawlpid" 2>/dev/null
|
||||
wait "$crawlpid" 2>/dev/null
|
||||
crawlpid=
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
|
||||
echo "FAIL: no temp-ref survived pass 1; cannot drive C7"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (temp-ref present)"
|
||||
before=$(wc -c <"$counter" 2>/dev/null || echo 0)
|
||||
|
||||
# --- pass 2: --continue -> resume Range -> bogus 304 -> drop + refetch --------
|
||||
printf '[pass 2: resume gets 304, must refetch] ..\t'
|
||||
httrack "${common[@]}" --continue "${base}/resume304/index.html" >"${tmpdir}/log2" 2>&1
|
||||
echo "OK (terminated)"
|
||||
|
||||
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
|
||||
full=8200 # len(RESUME304_BODY) = len("C7DATA--") + 8192
|
||||
|
||||
printf '[file recovered whole] ..\t'
|
||||
test -s "$blob" || {
|
||||
echo "FAIL: blob.bin missing after the 304 resume (partial dropped, not refetched)"
|
||||
exit 1
|
||||
}
|
||||
got=$(wc -c <"$blob")
|
||||
test "$got" -eq "$full" || {
|
||||
echo "FAIL: blob.bin is ${got} bytes, expected ${full} (partial finalized as complete)"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (${got} bytes)"
|
||||
|
||||
# Pass 2 must issue at least two requests: the resume (Range -> 304) and the
|
||||
# range-less refetch (-> 200). Exactly one means the partial was trusted.
|
||||
after=$(wc -c <"$counter" 2>/dev/null || echo 0)
|
||||
hits=$((after - before))
|
||||
printf '[resume then refetch] ..\t'
|
||||
test "$hits" -ge 2 || {
|
||||
echo "FAIL: only ${hits} pass-2 request(s); the 304 was trusted, no refetch"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (${hits} requests)"
|
||||
|
||||
# The recovery must go through the resume path: a Range request that drew the
|
||||
# bogus 304 (a fresh re-crawl sends no Range, so this marker stays empty).
|
||||
printf '[resume Range drew a 304] ..\t'
|
||||
test -s "$mark" || {
|
||||
echo "FAIL: no Range request got a 304; the resume path was not exercised"
|
||||
exit 1
|
||||
}
|
||||
echo "OK"
|
||||
@@ -1,96 +0,0 @@
|
||||
#!/bin/bash
|
||||
# C2: a resume whose 206 carries a Content-Range end of INT64_MAX must not
|
||||
# sign-overflow the crange+1 range check (UBSan aborts on the pre-fix binary).
|
||||
# Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets the hostile 206, and
|
||||
# must reject the range and refetch the whole file rather than overflow. Teeth
|
||||
# are UBSan-only: a normal build wraps to the same reject+restart.
|
||||
set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crange.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
|
||||
refdir="${out}/hts-cache/ref"
|
||||
|
||||
# --- pass 1: crawl, interrupt once the blob download is underway -------------
|
||||
printf '[pass 1: interrupt mid-download] ..\t'
|
||||
httrack "${common[@]}" "${base}/crange206/index.html" >"${tmpdir}/log1" 2>&1 &
|
||||
crawlpid=$!
|
||||
for _ in $(seq 1 300); do
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
|
||||
kill -0 "$crawlpid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
sleep 0.3
|
||||
kill -TERM "$crawlpid" 2>/dev/null
|
||||
wait "$crawlpid" 2>/dev/null
|
||||
crawlpid=
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
|
||||
echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (temp-ref present)"
|
||||
|
||||
# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
|
||||
printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
|
||||
httrack "${common[@]}" --continue "${base}/crange206/index.html" >"${tmpdir}/log2" 2>&1
|
||||
echo "OK (terminated)"
|
||||
|
||||
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
|
||||
full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
|
||||
|
||||
printf '[file recovered whole] ..\t'
|
||||
test -s "$blob" || {
|
||||
echo "FAIL: blob.bin missing after the hostile 206"
|
||||
exit 1
|
||||
}
|
||||
got=$(wc -c <"$blob")
|
||||
test "$got" -eq "$full" || {
|
||||
echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (${got} bytes)"
|
||||
@@ -1,96 +0,0 @@
|
||||
#!/bin/bash
|
||||
# C2 (memory branch): a resume whose 206 lies text/html with a matching
|
||||
# INT64_MAX Content-Length must not overflow the resume buffer-size add
|
||||
# (UBSan aborts on the pre-fix binary at alloc_mem += totalsize). Pass 1 leaves
|
||||
# a partial + temp-ref; pass 2 resumes and must reject and refetch the whole
|
||||
# file. Teeth are UBSan-only: a normal build wraps to the same reject+restart.
|
||||
set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangemem.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
|
||||
refdir="${out}/hts-cache/ref"
|
||||
|
||||
# --- pass 1: crawl, interrupt once the blob download is underway -------------
|
||||
printf '[pass 1: interrupt mid-download] ..\t'
|
||||
httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
|
||||
crawlpid=$!
|
||||
for _ in $(seq 1 300); do
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
|
||||
kill -0 "$crawlpid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
sleep 0.3
|
||||
kill -TERM "$crawlpid" 2>/dev/null
|
||||
wait "$crawlpid" 2>/dev/null
|
||||
crawlpid=
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
|
||||
echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (temp-ref present)"
|
||||
|
||||
# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
|
||||
printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
|
||||
httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1
|
||||
echo "OK (terminated)"
|
||||
|
||||
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
|
||||
full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
|
||||
|
||||
printf '[file recovered whole] ..\t'
|
||||
test -s "$blob" || {
|
||||
echo "FAIL: blob.bin missing after the hostile 206"
|
||||
exit 1
|
||||
}
|
||||
got=$(wc -c <"$blob")
|
||||
test "$got" -eq "$full" || {
|
||||
echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (${got} bytes)"
|
||||
@@ -30,7 +30,6 @@ TESTS = \
|
||||
01_engine-cmdline.test \
|
||||
01_engine-cookies.test \
|
||||
01_engine-copyopt.test \
|
||||
01_engine-crange.test \
|
||||
01_engine-dns.test \
|
||||
01_engine-doitlog.test \
|
||||
01_engine-entities.test \
|
||||
@@ -113,11 +112,6 @@ TESTS = \
|
||||
40_local-why.test \
|
||||
41_local-utf8-link.test \
|
||||
42_local-maxsize-slow.test \
|
||||
43_local-update-truncate.test \
|
||||
44_local-update-errormask.test \
|
||||
45_local-maxsize-recv.test \
|
||||
46_local-update-304-resume.test \
|
||||
47_local-crange-overflow.test \
|
||||
48_local-crange-memresume.test
|
||||
43_local-update-truncate.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -105,7 +105,7 @@ function start-crawl {
|
||||
log="${tmp}/log"
|
||||
debug starting httrack -O "${tmp}" "${moreargs[@]}" "${@:pos}"
|
||||
info "running httrack ${*:pos}"
|
||||
httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -mrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
|
||||
httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
|
||||
crawlpid="$!"
|
||||
debug "started cralwer on pid $crawlpid"
|
||||
wait "$crawlpid"
|
||||
|
||||
@@ -14,13 +14,11 @@
|
||||
# Usage:
|
||||
# bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
|
||||
# [--rerun-args 'ARGS'] \
|
||||
# --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
|
||||
# --errors N --files N --found PATH ... --directory PATH ... \
|
||||
# --log-found REGEX ... --log-not-found REGEX ... \
|
||||
# --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
|
||||
# --file-min-bytes PATH N --max-mirror-bytes N \
|
||||
# httrack BASEURL/some/path [httrack-args...]
|
||||
# --errors counts every "Error:" log line; --errors-content drops transient
|
||||
# network failures (codes -2..-6) that flake on busy loopback under -c8.
|
||||
# --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
|
||||
# --max/--min-mirror-bytes bound the mirrored content bytes (host root).
|
||||
# --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
|
||||
@@ -133,7 +131,7 @@ while test "$pos" -lt "$nargs"; do
|
||||
pos=$((pos + 1))
|
||||
rerun_args="${args[$pos]}"
|
||||
;;
|
||||
--errors | --errors-content | --files)
|
||||
--errors | --files)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
;;
|
||||
@@ -213,7 +211,7 @@ mkdir "$out" || die "could not create $out"
|
||||
declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
|
||||
log="${tmpdir}/log"
|
||||
info "running httrack ${hts[*]}"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
@@ -230,7 +228,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
|
||||
# --- optional second pass: re-mirror into the same dir (cache/update path) ----
|
||||
if test -n "$rerun"; then
|
||||
info "re-running httrack (update pass)"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
@@ -258,7 +256,7 @@ fi
|
||||
if test -n "$rerun_args"; then
|
||||
read -ra extra <<<"$rerun_args"
|
||||
info "re-running httrack with ${rerun_args}"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
@@ -281,7 +279,7 @@ if test -n "$rerun_dead"; then
|
||||
wait "$serverpid" 2>/dev/null
|
||||
serverpid=
|
||||
info "re-running httrack against the stopped server"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid" || true
|
||||
@@ -336,14 +334,6 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
assert_equals "checking errors" "${audit[$i]}" \
|
||||
"$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
|
||||
;;
|
||||
--errors-content)
|
||||
i=$((i + 1))
|
||||
total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
|
||||
# transient network failures (statuscode -2..-6) flake on busy loopback;
|
||||
# the code parens are followed by " at link" or " after N retries at link"
|
||||
transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
|
||||
assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
|
||||
;;
|
||||
--files)
|
||||
i=$((i + 1))
|
||||
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
|
||||
|
||||
@@ -790,56 +790,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
# C7: stall the first GET (partial + temp-ref), then answer the resume's
|
||||
# Range with a bogus 304; httrack must drop the partial and refetch.
|
||||
RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
|
||||
_resume304_started = False
|
||||
|
||||
def route_resume304_index(self):
|
||||
self.send_html('\t<a href="blob.bin">blob</a>')
|
||||
|
||||
def route_resume304(self):
|
||||
counter = os.environ.get("RESUME304_COUNTER")
|
||||
if counter:
|
||||
with open(counter, "a") as fp:
|
||||
fp.write("x")
|
||||
rng = self.headers.get("Range")
|
||||
if not Handler._resume304_started:
|
||||
Handler._resume304_started = True
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
self.send_header("Last-Modified", BIG_LASTMOD)
|
||||
self.send_header("ETag", '"c7"')
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.RESUME304_BODY[:4096])
|
||||
self.wfile.flush()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
if rng is not None: # resume request: bogus out-of-protocol 304
|
||||
mark = os.environ.get("RESUME304_MARK")
|
||||
if mark:
|
||||
with open(mark, "a") as fp:
|
||||
fp.write("z")
|
||||
self.send_response(304)
|
||||
self.send_header("ETag", '"c7"')
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
# Range-less refetch after the partial is dropped: whole file.
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.RESUME304_BODY)
|
||||
|
||||
# 206 resume must honor the server's Content-Range, not the offset we asked
|
||||
# for (#198): a server resuming a few bytes *before* the request must not
|
||||
# leave httrack duplicating the overlap onto the partial. flaky.bin
|
||||
@@ -905,94 +855,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(part)
|
||||
|
||||
# C2: a resume answered with a 206 whose Content-Range end is INT64_MAX would
|
||||
# sign-overflow the crange+1 range check (UBSan abort). Stall first (partial +
|
||||
# ref), then answer the resume Range with that hostile 206; httrack must reject
|
||||
# the range and refetch, never overflow.
|
||||
CRANGE206_BODY = b"CR206DAT" + bytes((i * 5 + 1) % 256 for i in range(6000))
|
||||
_crange206_started = False
|
||||
|
||||
def route_crange206_index(self):
|
||||
self.send_html('\t<a href="blob.bin">blob</a>')
|
||||
|
||||
def route_crange206(self):
|
||||
rng = self.headers.get("Range")
|
||||
if not Handler._crange206_started:
|
||||
Handler._crange206_started = True
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY[:3000])
|
||||
self.wfile.flush()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
if rng is not None: # resume: hostile 206, Content-Range end = INT64_MAX
|
||||
self.send_response(206, "Partial Content")
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.send_header("Content-Range", "bytes 0-9223372036854775807/1")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY)
|
||||
return
|
||||
# range-less refetch after the bad range is rejected: whole file.
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY)
|
||||
|
||||
# C2 memory branch: a resume answered with a 206 that lies text/html (so the
|
||||
# resume buffers in memory) plus a matching INT64_MAX Content-Length would
|
||||
# overflow the buffer-size add. Stall first, then send that hostile 206.
|
||||
_crange206mem_started = False
|
||||
|
||||
def route_crange206mem_index(self):
|
||||
self.send_html('\t<a href="blob.bin">blob</a>')
|
||||
|
||||
def route_crange206mem(self):
|
||||
rng = self.headers.get("Range")
|
||||
if not Handler._crange206mem_started:
|
||||
Handler._crange206mem_started = True
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY[:3000])
|
||||
self.wfile.flush()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
if rng is not None: # resume: text/html + matching INT64_MAX Content-Length
|
||||
self.send_response(206, "Partial Content")
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", "9223372036854775807")
|
||||
self.send_header(
|
||||
"Content-Range", "bytes 0-9223372036854775806/9223372036854775807"
|
||||
)
|
||||
self.end_headers()
|
||||
return # the overflow is computed before any body read
|
||||
# range-less refetch after the resume is rejected: whole file.
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY)
|
||||
|
||||
# error pages / 0-byte files (#17): -o0 ("no error pages") must keep 4xx/5xx
|
||||
# bodies off disk; a genuine 0-byte 200 is a valid file and stays.
|
||||
def route_errpage_index(self):
|
||||
@@ -1072,33 +934,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
def route_mini304_page(self):
|
||||
self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
|
||||
|
||||
# --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
|
||||
# on the update fetch must keep its good copy, not be overwritten nor purged.
|
||||
ERRMASK_GOOD = b"KEEP" + b"." * 1020 # 1024 B distinctive non-HTML body
|
||||
ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
|
||||
|
||||
def route_errmask_index(self):
|
||||
self.send_html('\t<a href="keep.dat">keep</a>\n')
|
||||
|
||||
def route_errmask_keep(self):
|
||||
# First crawl (no validator) gets the 1024 B body + Last-Modified; the
|
||||
# update sends a conditional and gets a 403 error page.
|
||||
if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
|
||||
self.send_response(403, "Forbidden")
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.ERRMASK_ERR)
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Last-Modified", BIG_LASTMOD)
|
||||
self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.ERRMASK_GOOD)
|
||||
|
||||
# --- delayed-type degenerate paths (issues #5/#107) --------------------
|
||||
def route_delayed_index(self):
|
||||
self.send_html(
|
||||
@@ -1242,22 +1077,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# -M received-volume cap (#520): links to large 404 bodies. httrack receives
|
||||
# each (HTS_TOTAL_RECV climbs) but saves none, so saved stays far below -M.
|
||||
def route_maxrecv_index(self):
|
||||
self.send_html(
|
||||
"".join('\t<a href="r%d.bin">r%d</a>\n' % (i, i) for i in range(16))
|
||||
)
|
||||
|
||||
def route_maxrecv_404(self):
|
||||
body = b"x" * self.BIGFILE_BYTES
|
||||
self.send_response(404, "Not Found")
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
ROUTES = {
|
||||
"/cookies/entrance.php": route_entrance,
|
||||
"/cookies/second.php": route_second,
|
||||
@@ -1287,15 +1106,9 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/intl/" + INTL_NAME: route_intl_page,
|
||||
"/resume/index.html": route_resume_index,
|
||||
"/resume/blob.txt": route_resume,
|
||||
"/resume304/index.html": route_resume304_index,
|
||||
"/resume304/blob.bin": route_resume304,
|
||||
"/overlap/index.html": route_overlap_index,
|
||||
"/overlap/flaky.bin": route_overlap,
|
||||
"/overlap/full.bin": route_overlap_full,
|
||||
"/crange206/index.html": route_crange206_index,
|
||||
"/crange206/blob.bin": route_crange206,
|
||||
"/crange206mem/index.html": route_crange206mem_index,
|
||||
"/crange206mem/blob.bin": route_crange206mem,
|
||||
"/size/index.html": route_size_index,
|
||||
"/size/oversize.bin": route_size_oversize,
|
||||
"/errpage/index.html": route_errpage_index,
|
||||
@@ -1364,25 +1177,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/redir/target.html": route_redir_target,
|
||||
"/mini304/index.html": route_mini304_index,
|
||||
"/mini304/page.html": route_mini304_page,
|
||||
"/errmask/index.html": route_errmask_index,
|
||||
"/errmask/keep.dat": route_errmask_keep,
|
||||
"/maxrecv/index.html": route_maxrecv_index,
|
||||
"/maxrecv/r0.bin": route_maxrecv_404,
|
||||
"/maxrecv/r1.bin": route_maxrecv_404,
|
||||
"/maxrecv/r2.bin": route_maxrecv_404,
|
||||
"/maxrecv/r3.bin": route_maxrecv_404,
|
||||
"/maxrecv/r4.bin": route_maxrecv_404,
|
||||
"/maxrecv/r5.bin": route_maxrecv_404,
|
||||
"/maxrecv/r6.bin": route_maxrecv_404,
|
||||
"/maxrecv/r7.bin": route_maxrecv_404,
|
||||
"/maxrecv/r8.bin": route_maxrecv_404,
|
||||
"/maxrecv/r9.bin": route_maxrecv_404,
|
||||
"/maxrecv/r10.bin": route_maxrecv_404,
|
||||
"/maxrecv/r11.bin": route_maxrecv_404,
|
||||
"/maxrecv/r12.bin": route_maxrecv_404,
|
||||
"/maxrecv/r13.bin": route_maxrecv_404,
|
||||
"/maxrecv/r14.bin": route_maxrecv_404,
|
||||
"/maxrecv/r15.bin": route_maxrecv_404,
|
||||
}
|
||||
|
||||
# --- /big/ seeded pseudo-site ------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user