mirror of
https://github.com/xroche/httrack.git
synced 2026-07-12 03:46:58 +03:00
Compare commits
5 Commits
test-176-e
...
fix-test19
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee550c3972 | ||
|
|
145ea4769f | ||
|
|
c5da5a2f2e | ||
|
|
6eaefe6233 | ||
|
|
33249be5a0 |
@@ -56,6 +56,10 @@ Please visit our Website: http://www.httrack.com
|
||||
// Platform detection (sizes, feature macros)
|
||||
#include "htsconfig.h"
|
||||
|
||||
// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
|
||||
#include <stdint.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
// WIN32 types
|
||||
#ifdef _WIN32
|
||||
#ifndef SIZEOF_LONG
|
||||
@@ -111,29 +115,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_DO_NOT_USE_UID
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef SIZEOF_LONG_LONG
|
||||
#if SIZEOF_LONG_LONG == 8
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef __sun
|
||||
#define HTS_LONGLONG 0
|
||||
#endif
|
||||
#ifdef __osf__
|
||||
#define HTS_LONGLONG 0
|
||||
#endif
|
||||
#ifdef __linux
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#ifdef _WIN32
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef DLLIB
|
||||
#define HTS_DLOPEN 1
|
||||
#else
|
||||
@@ -326,66 +307,11 @@ typedef int hts_tristate;
|
||||
#define HTS_DEPRECATED(msg)
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef HTS_NO_64_BIT
|
||||
#define HTS_LONGLONG 0
|
||||
#else
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* Wide integer types, chosen per platform.
|
||||
LLint: signed 64-bit counter for byte counts and large sizes (falls back to
|
||||
plain int where 64-bit is unavailable).
|
||||
TStamp: timestamp/duration in the same width (a double in the no-64-bit
|
||||
fallback).
|
||||
LLintP: the printf conversion for an LLint. */
|
||||
#if HTS_LONGLONG
|
||||
#ifdef LLINT_FORMAT
|
||||
typedef LLINT_TYPE LLint;
|
||||
|
||||
typedef LLINT_TYPE TStamp;
|
||||
|
||||
#define LLintP LLINT_FORMAT
|
||||
#else
|
||||
|
||||
#ifdef _WIN32
|
||||
typedef __int64 LLint;
|
||||
|
||||
typedef __int64 TStamp;
|
||||
|
||||
#define LLintP "%I64d"
|
||||
/* x32/ILP32 sets __x86_64__ but long is 32-bit; exclude it so LLint stays
|
||||
* 64-bit. */
|
||||
#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) || \
|
||||
defined(__64BIT__)) && \
|
||||
!defined(__ILP32__)
|
||||
|
||||
typedef long int LLint;
|
||||
|
||||
typedef long int TStamp;
|
||||
|
||||
#define LLintP "%ld"
|
||||
#else
|
||||
typedef long long int LLint;
|
||||
|
||||
typedef long long int TStamp;
|
||||
|
||||
#define LLintP "%lld"
|
||||
#endif
|
||||
|
||||
#endif /* HTS_LONGLONG */
|
||||
|
||||
/* Claiming long-long support must yield a real 64-bit LLint (x32 regressed:
|
||||
__x86_64__ set but long is 32-bit). Compile-time trip, portable to C90. */
|
||||
typedef char hts_assert_llint_is_64bit[sizeof(LLint) == 8 ? 1 : -1];
|
||||
|
||||
#else
|
||||
typedef int LLint;
|
||||
|
||||
#define LLintP "%d"
|
||||
typedef double TStamp;
|
||||
#endif
|
||||
/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
|
||||
typedef int64_t LLint;
|
||||
typedef int64_t TStamp;
|
||||
/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
|
||||
#define LLintP "%" PRId64
|
||||
|
||||
/* Integer type for file offsets/sizes passed to the C library. Widens to
|
||||
LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
|
||||
|
||||
22
src/htslib.c
22
src/htslib.c
@@ -3038,12 +3038,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
|
||||
/* See http://physics.nist.gov/cuu/Units/binary.html */
|
||||
#define ToLLint(a) ((LLint)(a))
|
||||
#define ToLLintKiB (ToLLint(1024))
|
||||
#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
|
||||
#ifdef HTS_LONGLONG
|
||||
#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#endif
|
||||
#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintPiB \
|
||||
(ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
if (n < ToLLintKiB) {
|
||||
sprintf(strc->buff1, "%d", (int) (LLint) n);
|
||||
@@ -3052,9 +3051,7 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
|
||||
(int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
|
||||
strcpybuff(strc->buff2, "KiB");
|
||||
}
|
||||
#ifdef HTS_LONGLONG
|
||||
else if (n < ToLLintGiB) {
|
||||
} else if (n < ToLLintGiB) {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
|
||||
(int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
|
||||
strcpybuff(strc->buff2, "MiB");
|
||||
@@ -3071,13 +3068,6 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
(int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
|
||||
strcpybuff(strc->buff2, "PiB");
|
||||
}
|
||||
#else
|
||||
else {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
|
||||
(int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
|
||||
strcpybuff(strc->buff2, "MiB");
|
||||
}
|
||||
#endif
|
||||
strc->buffadr[0] = strc->buff1;
|
||||
strc->buffadr[1] = strc->buff2;
|
||||
return strc->buffadr;
|
||||
|
||||
@@ -66,9 +66,12 @@ test -n "$port" || {
|
||||
}
|
||||
|
||||
out="$tmpdir/crawl"
|
||||
# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
|
||||
# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
|
||||
# bound the stall without changing what the fallback exercises.
|
||||
HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
|
||||
httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
|
||||
-c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
|
||||
-c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
|
||||
|
||||
log="$out/hts-log.txt"
|
||||
|
||||
@@ -92,12 +95,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# every address refused: the slot exhausts the list, then errors out (the
|
||||
# harness timeout would catch a hang/loop; refused connects are instant)
|
||||
# every address dead: the slot exhausts the list, then errors out (the harness
|
||||
# timeout would catch a hang/loop)
|
||||
out2="$tmpdir/crawl2"
|
||||
HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
|
||||
httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
|
||||
-c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
|
||||
-c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
|
||||
log2="$out2/hts-log.txt"
|
||||
|
||||
grep -q "trying next address" "$log2" || {
|
||||
|
||||
@@ -17,7 +17,10 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
|
||||
printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
|
||||
>"$tmproot/javaclass/Foo.class"
|
||||
|
||||
# --log-found first: a plugin-load failure (wrong soname, shadowing system
|
||||
# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
|
||||
--log-found '\+libhtsjava' \
|
||||
--found 'javaclass/Foo.class' \
|
||||
--found 'javaclass/hello.gif' \
|
||||
httrack 'BASEURL/javaclass/index.html'
|
||||
|
||||
@@ -9,7 +9,7 @@ set -euo pipefail
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --rerun \
|
||||
--errors 4 --files 361 \
|
||||
--errors-content 4 --files 361 \
|
||||
--found 'big/p/95.html' \
|
||||
--found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
|
||||
--found 'big/a/f2-2x.png' \
|
||||
|
||||
@@ -14,11 +14,13 @@
|
||||
# Usage:
|
||||
# bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
|
||||
# [--rerun-args 'ARGS'] \
|
||||
# --errors N --files N --found PATH ... --directory PATH ... \
|
||||
# --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
|
||||
# --log-found REGEX ... --log-not-found REGEX ... \
|
||||
# --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
|
||||
# --file-min-bytes PATH N --max-mirror-bytes N \
|
||||
# httrack BASEURL/some/path [httrack-args...]
|
||||
# --errors counts every "Error:" log line; --errors-content drops transient
|
||||
# network failures (codes -2..-6) that flake on busy loopback under -c8.
|
||||
# --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
|
||||
# --max/--min-mirror-bytes bound the mirrored content bytes (host root).
|
||||
# --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
|
||||
@@ -131,7 +133,7 @@ while test "$pos" -lt "$nargs"; do
|
||||
pos=$((pos + 1))
|
||||
rerun_args="${args[$pos]}"
|
||||
;;
|
||||
--errors | --files)
|
||||
--errors | --errors-content | --files)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
;;
|
||||
@@ -334,6 +336,14 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
assert_equals "checking errors" "${audit[$i]}" \
|
||||
"$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
|
||||
;;
|
||||
--errors-content)
|
||||
i=$((i + 1))
|
||||
total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
|
||||
# transient network failures (statuscode -2..-6) flake on busy loopback;
|
||||
# the code parens are followed by " at link" or " after N retries at link"
|
||||
transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
|
||||
assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
|
||||
;;
|
||||
--files)
|
||||
i=$((i + 1))
|
||||
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
|
||||
|
||||
Reference in New Issue
Block a user