mirror of
https://github.com/xroche/httrack.git
synced 2026-06-19 00:33:03 +03:00
Compare commits
4 Commits
feature/ap
...
fix/travel
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee6beeeb7d | ||
|
|
6788bda380 | ||
|
|
7ead8d595e | ||
|
|
93f502990c |
@@ -1991,7 +1991,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
case 'v':
|
||||
opt->verbosedisplay = 2;
|
||||
if (isdigit((unsigned char) *(com + 1))) {
|
||||
sscanf(com + 1, "%d", &opt->verbosedisplay);
|
||||
sscanf(com + 1, "%d", (int *) &opt->verbosedisplay);
|
||||
while(isdigit((unsigned char) *(com + 1)))
|
||||
com++;
|
||||
}
|
||||
@@ -2006,7 +2006,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
case 'N':
|
||||
opt->savename_delayed = 2;
|
||||
if (isdigit((unsigned char) *(com + 1))) {
|
||||
sscanf(com + 1, "%d", &opt->savename_delayed);
|
||||
sscanf(com + 1, "%d", (int *) &opt->savename_delayed);
|
||||
while(isdigit((unsigned char) *(com + 1)))
|
||||
com++;
|
||||
}
|
||||
|
||||
38
src/htsopt.h
38
src/htsopt.h
@@ -342,17 +342,37 @@ typedef enum hts_seeker {
|
||||
HTS_SEEKER_UP = 1 << 1 /**< may ascend to parent directories */
|
||||
} hts_seeker;
|
||||
|
||||
/* Link-following scope, stored in the low byte of opt->travel. */
|
||||
/* opt->travel: link-following scope in the low byte, flags OR'd in above it. */
|
||||
typedef enum hts_travel_scope {
|
||||
HTS_TRAVEL_SAME_ADDRESS = 0, /**< stay on the same address (host) */
|
||||
HTS_TRAVEL_SAME_DOMAIN = 1, /**< stay on the same principal domain */
|
||||
HTS_TRAVEL_SAME_TLD = 2, /**< stay on the same TLD (e.g. .com) */
|
||||
HTS_TRAVEL_EVERYWHERE = 7 /**< follow links anywhere on the web */
|
||||
HTS_TRAVEL_EVERYWHERE = 7, /**< follow links anywhere on the web */
|
||||
HTS_TRAVEL_TEST_ALL = 1 << 8 /**< also test forbidden URLs (-t) */
|
||||
} hts_travel_scope;
|
||||
|
||||
/* Flags OR'd into opt->travel above the scope value. */
|
||||
#define HTS_TRAVEL_SCOPE_MASK 0xff /**< mask selecting the scope value */
|
||||
#define HTS_TRAVEL_TEST_ALL (1 << 8) /**< also test forbidden URLs (-t) */
|
||||
/* Mask selecting the scope value out of opt->travel. */
|
||||
#define HTS_TRAVEL_SCOPE_MASK 0xff
|
||||
|
||||
/* Text progress display detail (opt->verbosedisplay). */
|
||||
typedef enum hts_verbosedisplay {
|
||||
HTS_VERBOSE_NONE = 0, /**< no animated progress display (default) */
|
||||
HTS_VERBOSE_SIMPLE = 1, /**< minimal single-line progress */
|
||||
HTS_VERBOSE_FULL = 2 /**< full animated progress */
|
||||
} hts_verbosedisplay;
|
||||
|
||||
/* Delayed file-type resolution policy (opt->savename_delayed). */
|
||||
typedef enum hts_savename_delayed {
|
||||
HTS_SAVENAME_DELAYED_NONE = 0, /**< resolve the type immediately */
|
||||
HTS_SAVENAME_DELAYED_SOFT = 1, /**< delay the type check when unknown */
|
||||
HTS_SAVENAME_DELAYED_HARD = 2 /**< always delay the type check (default) */
|
||||
} hts_savename_delayed;
|
||||
|
||||
/* Host-banning triggers (opt->hostcontrol bitmask). */
|
||||
typedef enum hts_hostcontrol {
|
||||
HTS_HOSTCONTROL_BAN_TIMEOUT = 1 << 0, /**< ban a timing-out host */
|
||||
HTS_HOSTCONTROL_BAN_SLOW = 1 << 1 /**< ban a too-slow host */
|
||||
} hts_hostcontrol;
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_lien_buffers
|
||||
#define HTS_DEF_FWSTRUCT_lien_buffers
|
||||
@@ -386,7 +406,7 @@ struct httrackp {
|
||||
hts_urlmode
|
||||
urlmode; /**< saved-link rewriting style (relative, absolute, etc.) */
|
||||
hts_boolean no_type_change; // do not change file type according to MIME
|
||||
int debug; /**< debug logging level */
|
||||
hts_log_type debug; /**< debug logging level */
|
||||
int getmode; /**< what to fetch (HTML, images, ...) bitmask */
|
||||
FILE *log; /**< informational log stream; NULL mutes it */
|
||||
FILE *errlog; /**< error log stream; NULL mutes it */
|
||||
@@ -414,7 +434,7 @@ struct httrackp {
|
||||
int savename_type; /**< saved-name layout (original tree, flat, ...) */
|
||||
String
|
||||
savename_userdef; /**< user-defined name template (e.g. %h%p/%n%q.%t) */
|
||||
int savename_delayed; // delayed type check
|
||||
hts_savename_delayed savename_delayed; /**< delayed type-check policy */
|
||||
hts_boolean
|
||||
delayed_cached; // delayed type check can be cached to speedup updates
|
||||
hts_boolean mimehtml; /**< produce a single MIME/MHTML archive */
|
||||
@@ -430,7 +450,7 @@ struct httrackp {
|
||||
hts_boolean makestat; /**< maintain a transfer-statistics log */
|
||||
hts_boolean maketrack; /**< maintain an operations-statistics log */
|
||||
int parsejava; /**< Java/JS parsing mode; see htsparsejava_flags */
|
||||
int hostcontrol; /**< drop hosts that are too slow, etc. */
|
||||
int hostcontrol; /**< ban slow/timing-out hosts; see hts_hostcontrol bits */
|
||||
hts_boolean errpage; /**< generate an error page on 404 and similar */
|
||||
hts_boolean
|
||||
check_type; /**< probe unknown-type links (cgi/asp/dir) and follow moves
|
||||
@@ -455,7 +475,7 @@ struct httrackp {
|
||||
parseall; /**< parse aggressively, including unknown tags with links */
|
||||
hts_boolean parsedebug; /**< parser debug mode */
|
||||
hts_boolean norecatch; /**< do not re-fetch files the user deleted locally */
|
||||
int verbosedisplay; /**< animated text progress display */
|
||||
hts_verbosedisplay verbosedisplay; /**< animated text progress display */
|
||||
String footer; /**< footer/info line injected into pages */
|
||||
int maxcache; /**< in-memory cache backing limit (bytes) */
|
||||
// int maxcache_anticipate; // maximum links to anticipate (upper bound)
|
||||
|
||||
@@ -3722,7 +3722,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
//case -1: can_retry=1; break;
|
||||
case STATUSCODE_TIMEOUT:
|
||||
if (opt->hostcontrol) { // timeout et retry épuisés
|
||||
if ((opt->hostcontrol & 1) && (heap(ptr)->retry <= 0)) {
|
||||
if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) &&
|
||||
(heap(ptr)->retry <= 0)) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", urladr(), urlfil());
|
||||
host_ban(opt, ptr, sback, jump_identification_const(urladr()));
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
@@ -3735,7 +3736,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
break;
|
||||
case STATUSCODE_SLOW:
|
||||
if ((opt->hostcontrol) && (heap(ptr)->retry <= 0)) { // too slow
|
||||
if (opt->hostcontrol & 2) {
|
||||
if (opt->hostcontrol & HTS_HOSTCONTROL_BAN_SLOW) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Link banned: %s%s", urladr(), urlfil());
|
||||
host_ban(opt, ptr, sback, jump_identification_const(urladr()));
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
|
||||
Reference in New Issue
Block a user