mirror of
https://github.com/xroche/httrack.git
synced 2026-07-12 11:57:30 +03:00
Compare commits
4 Commits
fix-176-up
...
llint-int6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e2e4e7d34 | ||
|
|
42c5e71f36 | ||
|
|
167ede32c0 | ||
|
|
c8b7c63c9b |
@@ -3540,22 +3540,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (back[i].r.statuscode == 406) { // 'Not Acceptable'
|
||||
back[i].r.statuscode = HTTP_OK;
|
||||
}
|
||||
// 'do not erase already downloaded file'
|
||||
// on an updated file
|
||||
// with an error : consider a 304 error
|
||||
if (!opt->delete_old) {
|
||||
if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
|
||||
&& !back[i].testmode) {
|
||||
if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"Error ignored %d (%s) because of 'no purge' option for %s%s",
|
||||
back[i].r.statuscode, back[i].r.msg,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
back[i].r.statuscode = HTTP_NOT_MODIFIED;
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
}
|
||||
}
|
||||
// On update, keep the good copy on error (mask as 304); skip
|
||||
// resume paths so a stale-partial 416 still re-fetches.
|
||||
if (HTTP_IS_ERROR(back[i].r.statuscode) &&
|
||||
back[i].is_update && !back[i].testmode &&
|
||||
back[i].range_req_size == 0 && back[i].url_sav[0] &&
|
||||
fexist_utf8(back[i].url_sav)) {
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"Error %d (%s) ignored on update, keeping "
|
||||
"existing copy: %s%s",
|
||||
back[i].r.statuscode, back[i].r.msg,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
back[i].r.statuscode = HTTP_NOT_MODIFIED;
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
}
|
||||
// Various hacks to limit re-transfers when updating a mirror
|
||||
// Force update if same size detected
|
||||
|
||||
@@ -56,6 +56,10 @@ Please visit our Website: http://www.httrack.com
|
||||
// Platform detection (sizes, feature macros)
|
||||
#include "htsconfig.h"
|
||||
|
||||
// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
|
||||
#include <stdint.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
// WIN32 types
|
||||
#ifdef _WIN32
|
||||
#ifndef SIZEOF_LONG
|
||||
@@ -111,29 +115,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_DO_NOT_USE_UID
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef SIZEOF_LONG_LONG
|
||||
#if SIZEOF_LONG_LONG == 8
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef __sun
|
||||
#define HTS_LONGLONG 0
|
||||
#endif
|
||||
#ifdef __osf__
|
||||
#define HTS_LONGLONG 0
|
||||
#endif
|
||||
#ifdef __linux
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#ifdef _WIN32
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef DLLIB
|
||||
#define HTS_DLOPEN 1
|
||||
#else
|
||||
@@ -326,59 +307,11 @@ typedef int hts_tristate;
|
||||
#define HTS_DEPRECATED(msg)
|
||||
#endif
|
||||
|
||||
#ifndef HTS_LONGLONG
|
||||
#ifdef HTS_NO_64_BIT
|
||||
#define HTS_LONGLONG 0
|
||||
#else
|
||||
#define HTS_LONGLONG 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* Wide integer types, chosen per platform.
|
||||
LLint: signed 64-bit counter for byte counts and large sizes (falls back to
|
||||
plain int where 64-bit is unavailable).
|
||||
TStamp: timestamp/duration in the same width (a double in the no-64-bit
|
||||
fallback).
|
||||
LLintP: the printf conversion for an LLint. */
|
||||
#if HTS_LONGLONG
|
||||
#ifdef LLINT_FORMAT
|
||||
typedef LLINT_TYPE LLint;
|
||||
|
||||
typedef LLINT_TYPE TStamp;
|
||||
|
||||
#define LLintP LLINT_FORMAT
|
||||
#else
|
||||
|
||||
#ifdef _WIN32
|
||||
typedef __int64 LLint;
|
||||
|
||||
typedef __int64 TStamp;
|
||||
|
||||
#define LLintP "%I64d"
|
||||
#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) || \
|
||||
defined(__64BIT__))
|
||||
|
||||
typedef long int LLint;
|
||||
|
||||
typedef long int TStamp;
|
||||
|
||||
#define LLintP "%ld"
|
||||
#else
|
||||
typedef long long int LLint;
|
||||
|
||||
typedef long long int TStamp;
|
||||
|
||||
#define LLintP "%lld"
|
||||
#endif
|
||||
|
||||
#endif /* HTS_LONGLONG */
|
||||
|
||||
#else
|
||||
typedef int LLint;
|
||||
|
||||
#define LLintP "%d"
|
||||
typedef double TStamp;
|
||||
#endif
|
||||
/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
|
||||
typedef int64_t LLint;
|
||||
typedef int64_t TStamp;
|
||||
/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
|
||||
#define LLintP "%" PRId64
|
||||
|
||||
/* Integer type for file offsets/sizes passed to the C library. Widens to
|
||||
LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
|
||||
|
||||
22
src/htslib.c
22
src/htslib.c
@@ -3038,12 +3038,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
|
||||
/* See http://physics.nist.gov/cuu/Units/binary.html */
|
||||
#define ToLLint(a) ((LLint)(a))
|
||||
#define ToLLintKiB (ToLLint(1024))
|
||||
#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
|
||||
#ifdef HTS_LONGLONG
|
||||
#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
|
||||
#endif
|
||||
#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
#define ToLLintPiB \
|
||||
(ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
|
||||
HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
if (n < ToLLintKiB) {
|
||||
sprintf(strc->buff1, "%d", (int) (LLint) n);
|
||||
@@ -3052,9 +3051,7 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
|
||||
(int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
|
||||
strcpybuff(strc->buff2, "KiB");
|
||||
}
|
||||
#ifdef HTS_LONGLONG
|
||||
else if (n < ToLLintGiB) {
|
||||
} else if (n < ToLLintGiB) {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
|
||||
(int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
|
||||
strcpybuff(strc->buff2, "MiB");
|
||||
@@ -3071,13 +3068,6 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
|
||||
(int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
|
||||
strcpybuff(strc->buff2, "PiB");
|
||||
}
|
||||
#else
|
||||
else {
|
||||
sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
|
||||
(int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
|
||||
strcpybuff(strc->buff2, "MiB");
|
||||
}
|
||||
#endif
|
||||
strc->buffadr[0] = strc->buff1;
|
||||
strc->buffadr[1] = strc->buff2;
|
||||
return strc->buffadr;
|
||||
|
||||
11
tests/44_local-update-errormask.test
Normal file
11
tests/44_local-update-errormask.test
Normal file
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
|
||||
# keep its good local copy, not be overwritten by the error body nor purged.
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --rerun \
|
||||
--found 'errmask/keep.dat' \
|
||||
--file-min-bytes 'errmask/keep.dat' 1024 \
|
||||
httrack 'BASEURL/errmask/index.html'
|
||||
@@ -112,6 +112,7 @@ TESTS = \
|
||||
40_local-why.test \
|
||||
41_local-utf8-link.test \
|
||||
42_local-maxsize-slow.test \
|
||||
43_local-update-truncate.test
|
||||
43_local-update-truncate.test \
|
||||
44_local-update-errormask.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -934,6 +934,33 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
def route_mini304_page(self):
|
||||
self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
|
||||
|
||||
# --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
|
||||
# on the update fetch must keep its good copy, not be overwritten nor purged.
|
||||
ERRMASK_GOOD = b"KEEP" + b"." * 1020 # 1024 B distinctive non-HTML body
|
||||
ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
|
||||
|
||||
def route_errmask_index(self):
|
||||
self.send_html('\t<a href="keep.dat">keep</a>\n')
|
||||
|
||||
def route_errmask_keep(self):
|
||||
# First crawl (no validator) gets the 1024 B body + Last-Modified; the
|
||||
# update sends a conditional and gets a 403 error page.
|
||||
if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
|
||||
self.send_response(403, "Forbidden")
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.ERRMASK_ERR)
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Last-Modified", BIG_LASTMOD)
|
||||
self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.ERRMASK_GOOD)
|
||||
|
||||
# --- delayed-type degenerate paths (issues #5/#107) --------------------
|
||||
def route_delayed_index(self):
|
||||
self.send_html(
|
||||
@@ -1177,6 +1204,8 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/redir/target.html": route_redir_target,
|
||||
"/mini304/index.html": route_mini304_index,
|
||||
"/mini304/page.html": route_mini304_page,
|
||||
"/errmask/index.html": route_errmask_index,
|
||||
"/errmask/keep.dat": route_errmask_keep,
|
||||
}
|
||||
|
||||
# --- /big/ seeded pseudo-site ------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user