mirror of
https://github.com/xroche/httrack.git
synced 2026-07-17 14:19:48 +03:00
Compare commits
10 Commits
fix-607-ss
...
fix-expand
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a22b7a0e54 | ||
|
|
5944feed9d | ||
|
|
223a8fc27a | ||
|
|
de09705732 | ||
|
|
96b0555522 | ||
|
|
9bbfa39d2a | ||
|
|
b10c6eaf5c | ||
|
|
7be61d3111 | ||
|
|
3cc49b79c3 | ||
|
|
959deb0afb |
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "14 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "16 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -165,7 +165,7 @@ pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\
|
||||
.IP \-cN
|
||||
number of multiple connections (*c8) (\-\-sockets[=N])
|
||||
.IP \-TN
|
||||
timeout, number of seconds after a non\-responding link is shutdown (\-\-timeout[=N])
|
||||
timeout, number of seconds after a non\-responding link is shutdown; also bounds host name resolution (\-\-timeout[=N])
|
||||
.IP \-RN
|
||||
number of retries, in case of timeout or non\-fatal errors (*R1) (\-\-retries[=N])
|
||||
.IP \-JN
|
||||
|
||||
@@ -567,24 +567,29 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Get home directory, '.' if failed */
|
||||
/* Get home directory, '.' if unset or empty */
|
||||
/* example: /home/smith */
|
||||
const char *hts_gethome(void) {
|
||||
const char *home = getenv("HOME");
|
||||
|
||||
if (home)
|
||||
return home;
|
||||
else
|
||||
return ".";
|
||||
/* An empty $HOME would expand ~/foo into the absolute /foo */
|
||||
return strnotempty(home) ? home : ".";
|
||||
}
|
||||
|
||||
/* Convert ~/foo into /home/smith/foo */
|
||||
/* Convert ~/foo into /home/smith/foo (~user/ left alone: no getpwnam here) */
|
||||
void expand_home(String * str) {
|
||||
if (StringSub(*str, 1) == '~') {
|
||||
if (StringNotEmpty(*str) && StringSub(*str, 0) == '~' &&
|
||||
(StringLength(*str) == 1 || StringSub(*str, 1) == '/')) {
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
const char *const home = hts_gethome();
|
||||
const size_t homelen = strlen(home);
|
||||
const size_t taillen = StringLength(*str) - 1;
|
||||
|
||||
strcpybuff(tempo, hts_gethome());
|
||||
strcatbuff(tempo, StringBuff(*str) + 1);
|
||||
StringCopy(*str, tempo);
|
||||
/* Leave untouched rather than abort() in strcatbuff on a huge $HOME */
|
||||
if (taillen < sizeof(tempo) && homelen < sizeof(tempo) - taillen) {
|
||||
strcpybuff(tempo, home);
|
||||
strcatbuff(tempo, StringBuff(*str) + 1);
|
||||
StringCopy(*str, tempo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,9 +63,14 @@ typedef struct mock_host {
|
||||
int gai_err; /* non-zero: getaddrinfo returns this */
|
||||
int naddr;
|
||||
mock_addr addr[6];
|
||||
int calls; /* times the backend resolved this host */
|
||||
int calls; /* times the backend resolved this host */
|
||||
int slow_ms; /* non-zero: block this long, as a black-hole resolver would */
|
||||
} mock_host;
|
||||
|
||||
/* Long enough to outlast the 1s --timeout the bounded resolve is checked
|
||||
against, short enough to keep the self-test quick. */
|
||||
#define MOCK_SLOW_MS 3000
|
||||
|
||||
static mock_host mock_hosts[] = {
|
||||
{"v4only.test", 0, 1, {{AF_INET, {1, 2, 3, 4}}}, 0},
|
||||
{"v6only.test", 0, 1, {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 1}}}, 0},
|
||||
@@ -95,8 +100,15 @@ static mock_host mock_hosts[] = {
|
||||
{AF_INET, {10, 0, 0, 6}}},
|
||||
0},
|
||||
{"nodns.test", EAI_NONAME, 0, {{0}}, 0},
|
||||
/* resolves, but only well after --timeout: the #606 wedge */
|
||||
{"slow.test", 0, 1, {{AF_INET, {127, 0, 0, 9}}}, 0, MOCK_SLOW_MS},
|
||||
};
|
||||
|
||||
/* Serializes mock_host bookkeeping: a timed-out resolve is abandoned, so its
|
||||
worker is still inside the backend while the test reads the counters. */
|
||||
static htsmutex mock_lock = HTSMUTEX_INIT;
|
||||
static int mock_finished = 0; /* backend calls that have returned */
|
||||
|
||||
static mock_host *mock_find(const char *name) {
|
||||
for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++) {
|
||||
if (strcmp(mock_hosts[i].name, name) == 0)
|
||||
@@ -106,8 +118,34 @@ static mock_host *mock_find(const char *name) {
|
||||
}
|
||||
|
||||
static void mock_reset_calls(void) {
|
||||
hts_mutexlock(&mock_lock);
|
||||
for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++)
|
||||
mock_hosts[i].calls = 0;
|
||||
mock_finished = 0;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
}
|
||||
|
||||
static int mock_read_calls(const char *name) {
|
||||
int calls;
|
||||
|
||||
hts_mutexlock(&mock_lock);
|
||||
calls = mock_find(name)->calls;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
return calls;
|
||||
}
|
||||
|
||||
/* Wait for n backend calls to return, ordering their writes against ours. */
|
||||
static void mock_wait_finished(int n) {
|
||||
for (;;) {
|
||||
hts_boolean done;
|
||||
|
||||
hts_mutexlock(&mock_lock);
|
||||
done = (mock_finished >= n) ? HTS_TRUE : HTS_FALSE;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
if (done)
|
||||
break;
|
||||
Sleep(10);
|
||||
}
|
||||
}
|
||||
|
||||
/* Build one addrinfo node owning its sockaddr (freed by mock_freeaddrinfo). */
|
||||
@@ -133,10 +171,10 @@ static struct addrinfo *mock_mkai(const mock_addr *a) {
|
||||
return ai;
|
||||
}
|
||||
|
||||
static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
const char *service,
|
||||
const struct addrinfo *hints,
|
||||
struct addrinfo **res) {
|
||||
static int HTS_RESOLVER_CALL mock_getaddrinfo_(const char *node,
|
||||
const char *service,
|
||||
const struct addrinfo *hints,
|
||||
struct addrinfo **res) {
|
||||
mock_host *const h = mock_find(node);
|
||||
const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
|
||||
struct addrinfo *head = NULL, *tail = NULL;
|
||||
@@ -145,7 +183,11 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
*res = NULL;
|
||||
if (h == NULL)
|
||||
return EAI_NONAME;
|
||||
hts_mutexlock(&mock_lock);
|
||||
h->calls++; /* a real backend hit; a cached host skips this */
|
||||
hts_mutexrelease(&mock_lock);
|
||||
if (h->slow_ms != 0)
|
||||
Sleep(h->slow_ms);
|
||||
if (h->gai_err != 0)
|
||||
return h->gai_err;
|
||||
for (int i = 0; i < h->naddr; i++) {
|
||||
@@ -165,6 +207,18 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
const char *service,
|
||||
const struct addrinfo *hints,
|
||||
struct addrinfo **res) {
|
||||
const int ret = mock_getaddrinfo_(node, service, hints, res);
|
||||
|
||||
hts_mutexlock(&mock_lock);
|
||||
mock_finished++;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void HTS_RESOLVER_CALL mock_freeaddrinfo(struct addrinfo *res) {
|
||||
while (res != NULL) {
|
||||
struct addrinfo *const next = res->ai_next;
|
||||
@@ -370,6 +424,91 @@ int dns_selftests(httrackp *opt) {
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* Probes how long acquiring opt->state.lock takes while a resolve is in
|
||||
flight. hts_has_stopped() takes that same lock and mutates nothing, and the
|
||||
API promises it stays callable from another thread during a mirror. */
|
||||
typedef struct lock_probe {
|
||||
httrackp *opt;
|
||||
htsmutex lock;
|
||||
hts_boolean done;
|
||||
TStamp blocked_ms;
|
||||
} lock_probe;
|
||||
|
||||
static void lock_probe_thread(void *arg) {
|
||||
lock_probe *const p = (lock_probe *) arg;
|
||||
TStamp start;
|
||||
|
||||
Sleep(MOCK_SLOW_MS / 10); /* let the resolve get under way first */
|
||||
start = mtime_local();
|
||||
(void) hts_has_stopped(p->opt);
|
||||
hts_mutexlock(&p->lock);
|
||||
p->blocked_ms = mtime_local() - start;
|
||||
p->done = HTS_TRUE;
|
||||
hts_mutexrelease(&p->lock);
|
||||
}
|
||||
|
||||
int dns_timeout_selftests(httrackp *opt) {
|
||||
SOCaddr addrs[HTS_MAXADDRNUM];
|
||||
const char *err = NULL;
|
||||
lock_probe probe;
|
||||
TStamp start, elapsed;
|
||||
int count;
|
||||
|
||||
failures = 0;
|
||||
hts_dns_set_resolver_backend(&mock_backend);
|
||||
IPV6_resolver = 0;
|
||||
mock_reset_calls();
|
||||
opt->timeout = 1; /* the bound under test */
|
||||
|
||||
memset(&probe, 0, sizeof(probe));
|
||||
probe.opt = opt;
|
||||
probe.lock = HTSMUTEX_INIT;
|
||||
hts_mutexinit(&probe.lock);
|
||||
CHECK(hts_newthread(lock_probe_thread, &probe) == 0);
|
||||
|
||||
start = mtime_local();
|
||||
count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
|
||||
elapsed = mtime_local() - start;
|
||||
|
||||
/* the resolve returns on opt->timeout, not when the resolver deigns to
|
||||
answer: this is what lets --max-time and --timeout fire (#606). The bound
|
||||
is derived from opt->timeout, never from the mock's sleep, or a resolve
|
||||
that ignored opt->timeout would still pass under the mock. */
|
||||
CHECK(elapsed < (TStamp) opt->timeout * 1000 + 500);
|
||||
CHECK(count == 0); /* a timeout is reported as "does not resolve" */
|
||||
|
||||
/* state.lock is not held across the resolve; a concurrent stop query, which
|
||||
the mirror API promises stays live, is not blocked behind it */
|
||||
for (;;) {
|
||||
hts_boolean done;
|
||||
TStamp blocked;
|
||||
|
||||
hts_mutexlock(&probe.lock);
|
||||
done = probe.done;
|
||||
blocked = probe.blocked_ms;
|
||||
hts_mutexrelease(&probe.lock);
|
||||
if (done) {
|
||||
CHECK(blocked < 500);
|
||||
break;
|
||||
}
|
||||
Sleep(20);
|
||||
}
|
||||
hts_mutexfree(&probe.lock);
|
||||
|
||||
/* a timeout is not an answer, so it must not be negative-cached: the host is
|
||||
resolved again rather than written off for the rest of the crawl */
|
||||
CHECK(mock_read_calls("slow.test") == 1);
|
||||
count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
|
||||
CHECK(count == 0);
|
||||
CHECK(mock_read_calls("slow.test") == 2); /* re-resolved, not cached */
|
||||
|
||||
/* Both resolves were abandoned mid-backend; wait for their workers to leave
|
||||
it before returning. The backend stays installed: an abandoned worker
|
||||
still reads it (to free its addrinfo) after the last call returns. */
|
||||
mock_wait_finished(2);
|
||||
return failures;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int dns_selftests(httrackp *opt) {
|
||||
|
||||
@@ -46,6 +46,12 @@ typedef struct httrackp httrackp;
|
||||
Returns the number of failed checks (0 == success). */
|
||||
int dns_selftests(httrackp *opt);
|
||||
|
||||
/* Drive a deliberately slow (mock) resolver, asserting that a resolve is
|
||||
bounded by opt->timeout, does not hold opt->state.lock while it runs, and
|
||||
does not cache a timeout as an answer (#606). Takes a few seconds.
|
||||
Returns the number of failed checks (0 == success). */
|
||||
int dns_timeout_selftests(httrackp *opt);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -496,8 +496,8 @@ void help(const char *app, int more) {
|
||||
infomsg("");
|
||||
infomsg("Flow control:");
|
||||
infomsg(" cN number of multiple connections (*c8)");
|
||||
infomsg
|
||||
(" TN timeout, number of seconds after a non-responding link is shutdown");
|
||||
infomsg(" TN timeout, number of seconds after a non-responding link is"
|
||||
" shutdown; also bounds host name resolution");
|
||||
infomsg
|
||||
(" RN number of retries, in case of timeout or non-fatal errors (*R1)");
|
||||
infomsg
|
||||
|
||||
203
src/htslib.c
203
src/htslib.c
@@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#endif /* _WIN32 */
|
||||
#include <stdarg.h>
|
||||
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <stdarg.h>
|
||||
@@ -3747,6 +3748,20 @@ static int proxy_default_port(const char *arg) {
|
||||
return hts_proxy_is_socks(arg) ? 1080 : 8080;
|
||||
}
|
||||
|
||||
// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
|
||||
// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
|
||||
static int parse_proxy_port(const char *a, const char *arg) {
|
||||
char *end;
|
||||
long p;
|
||||
|
||||
if (!isdigit((unsigned char) *a)) // strtol would eat a sign or leading space
|
||||
return proxy_default_port(arg);
|
||||
p = strtol(a, &end, 10);
|
||||
if (*end != '\0' || p < 1 || p > 65535) // ERANGE lands out of range too
|
||||
return proxy_default_port(arg);
|
||||
return (int) p;
|
||||
}
|
||||
|
||||
void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
const char *authority = strstr(arg, "://");
|
||||
const char *a;
|
||||
@@ -3762,10 +3777,7 @@ void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
while (a > authority && a[-1] != ':' && a[-1] != '@' && a[-1] != ']')
|
||||
a--;
|
||||
if (a > authority && a[-1] == ':') {
|
||||
int p = -1;
|
||||
|
||||
sscanf(a, "%d", &p);
|
||||
*port = (p > 0) ? p : proxy_default_port(arg);
|
||||
*port = parse_proxy_port(a, arg);
|
||||
namelen = (size_t) (a - 1 - arg);
|
||||
} else {
|
||||
*port = proxy_default_port(arg);
|
||||
@@ -5084,50 +5096,164 @@ HTSEXT_API int check_hostname_dns(const char *const hostname) {
|
||||
return hts_dns_resolve_nocache(hostname, &buffer) != NULL;
|
||||
}
|
||||
|
||||
// Needs locking
|
||||
// Internal DNS cache. Fill out[0..count-1] with up to max addresses for _iadr,
|
||||
// resolving (and caching the full list) on a miss. Returns the count.
|
||||
static int hts_dns_resolve_list_(httrackp *opt, const char *_iadr,
|
||||
SOCaddr *const out, const int max,
|
||||
const char **error) {
|
||||
char BIGSTK iadr[HTS_URLMAXSIZE * 2];
|
||||
coucal cache = hts_cache(opt); // le cache dns
|
||||
/* A resolve in flight. Refcounted: a timed-out resolve is abandoned, not
|
||||
cancelled, so the last of caller/worker to leave frees the job. */
|
||||
typedef struct dns_resolve_job {
|
||||
htsmutex lock;
|
||||
int refcount;
|
||||
hts_boolean done;
|
||||
char *hostname;
|
||||
SOCaddr addr[HTS_MAXADDRNUM];
|
||||
int count;
|
||||
const char *error;
|
||||
} dns_resolve_job;
|
||||
|
||||
/* Copy the first min(count, max) addresses of src into dest. */
|
||||
static void dns_copy_addrs(SOCaddr *dest, SOCaddr *src, int count, int max) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < count && i < max; i++)
|
||||
SOCaddr_copy_SOCaddr(dest[i], src[i]);
|
||||
}
|
||||
|
||||
static void dns_job_release(dns_resolve_job *job) {
|
||||
hts_boolean last;
|
||||
|
||||
hts_mutexlock(&job->lock);
|
||||
last = (--job->refcount == 0) ? HTS_TRUE : HTS_FALSE;
|
||||
hts_mutexrelease(&job->lock);
|
||||
if (last) {
|
||||
hts_mutexfree(&job->lock);
|
||||
freet(job->hostname);
|
||||
freet(job);
|
||||
}
|
||||
}
|
||||
|
||||
/* Outlives a timed-out resolve, so it writes only the job: never opt (freed
|
||||
before the thread wait at exit) nor the DNS cache. */
|
||||
static void dns_resolve_thread(void *arg) {
|
||||
dns_resolve_job *const job = (dns_resolve_job *) arg;
|
||||
SOCaddr resolved[HTS_MAXADDRNUM];
|
||||
const char *error = NULL;
|
||||
const int count = hts_dns_resolve_nocache_list(job->hostname, resolved,
|
||||
HTS_MAXADDRNUM, &error);
|
||||
|
||||
hts_mutexlock(&job->lock);
|
||||
dns_copy_addrs(job->addr, resolved, count, HTS_MAXADDRNUM);
|
||||
job->count = count;
|
||||
job->error = error;
|
||||
job->done = HTS_TRUE; /* published last: gates the caller's read of addr[] */
|
||||
hts_mutexrelease(&job->lock);
|
||||
dns_job_release(job);
|
||||
}
|
||||
|
||||
/* Resolve hostname on a worker thread, giving up after timeout seconds.
|
||||
Returns the address count, or -1 on timeout -- distinct from 0 ("does not
|
||||
resolve"), which is a real answer and gets negative-cached. */
|
||||
static int hts_dns_resolve_nocache_list_bounded(const char *hostname,
|
||||
SOCaddr *const out,
|
||||
const int max,
|
||||
const int timeout,
|
||||
const char **error) {
|
||||
dns_resolve_job *job;
|
||||
TStamp deadline;
|
||||
int count = -1;
|
||||
int poll_ms = 1;
|
||||
|
||||
if (timeout <= 0) /* no bound asked for (--timeout 0) */
|
||||
return hts_dns_resolve_nocache_list(hostname, out, max, error);
|
||||
|
||||
job = calloct(1, sizeof(*job));
|
||||
assertf(job != NULL);
|
||||
hts_mutexinit(&job->lock);
|
||||
job->hostname = strdupt(hostname);
|
||||
job->refcount = 2; /* this caller + the worker */
|
||||
if (hts_newthread(dns_resolve_thread, job) != 0) {
|
||||
job->refcount = 1; /* no worker: fall back to resolving inline */
|
||||
dns_job_release(job);
|
||||
return hts_dns_resolve_nocache_list(hostname, out, max, error);
|
||||
}
|
||||
|
||||
deadline = mtime_local() + (TStamp) timeout * 1000;
|
||||
for (;;) {
|
||||
hts_boolean done;
|
||||
|
||||
hts_mutexlock(&job->lock);
|
||||
done = job->done;
|
||||
if (done) {
|
||||
count = job->count;
|
||||
dns_copy_addrs(out, job->addr, count, max);
|
||||
if (error != NULL)
|
||||
*error = job->error;
|
||||
}
|
||||
hts_mutexrelease(&job->lock);
|
||||
if (done || mtime_local() >= deadline)
|
||||
break;
|
||||
Sleep(poll_ms);
|
||||
if (poll_ms < 50) /* short first polls keep a fast resolve fast */
|
||||
poll_ms *= 2;
|
||||
}
|
||||
dns_job_release(job);
|
||||
return count;
|
||||
}
|
||||
|
||||
int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
|
||||
const char **error) {
|
||||
char BIGSTK host[HTS_URLMAXSIZE * 2];
|
||||
SOCaddr resolved[HTS_MAXADDRNUM];
|
||||
coucal cache;
|
||||
int count, i;
|
||||
|
||||
assertf(opt != NULL);
|
||||
assertf(_iadr != NULL);
|
||||
assertf(out != NULL);
|
||||
if (!strnotempty(iadr) || max <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
strcpybuff(iadr, jump_identification_const(_iadr));
|
||||
// couper éventuel :
|
||||
/* cache key and resolver input: identification and any ":port" stripped */
|
||||
strcpybuff(host, jump_identification_const(iadr));
|
||||
{
|
||||
char *a;
|
||||
|
||||
if ((a = jump_toport(iadr)))
|
||||
if ((a = jump_toport(host)))
|
||||
*a = '\0';
|
||||
}
|
||||
|
||||
/* get IP from the dns cache */
|
||||
count = hts_ghbn_all(cache, iadr, out, max);
|
||||
hts_mutexlock(&opt->state.lock);
|
||||
cache = hts_cache(opt);
|
||||
#if HTS_INET6 != 0
|
||||
hts_resolver_check_env(); /* settle the backend before a worker reads it */
|
||||
#endif
|
||||
count = hts_ghbn_all(cache, host, out, max);
|
||||
hts_mutexrelease(&opt->state.lock);
|
||||
if (count >= 0) { // cache hit (0 == negative-cached)
|
||||
return count;
|
||||
} else { // non présent dans le cache dns, tester
|
||||
SOCaddr resolved[HTS_MAXADDRNUM];
|
||||
t_dnscache *record;
|
||||
int i;
|
||||
}
|
||||
|
||||
#if DEBUGDNS
|
||||
printf("resolving (not cached) %s\n", iadr);
|
||||
printf("resolving (not cached) %s\n", host);
|
||||
#endif
|
||||
|
||||
count = hts_dns_resolve_nocache_list(iadr, resolved, HTS_MAXADDRNUM, error);
|
||||
/* Resolve with no lock held: getaddrinfo can block for a long time, and
|
||||
state.lock also gates the stop request (#606). */
|
||||
count = hts_dns_resolve_nocache_list_bounded(host, resolved, HTS_MAXADDRNUM,
|
||||
opt->timeout, error);
|
||||
|
||||
#if HTS_WIDE_DEBUG
|
||||
DEBUG_W("gethostbyname done\n");
|
||||
DEBUG_W("gethostbyname done\n");
|
||||
#endif
|
||||
|
||||
/* attempt to store new entry (coucal owns it and dups the host key) */
|
||||
record = malloct(sizeof(t_dnscache));
|
||||
if (count < 0) { /* timed out: no answer to cache, and none to report */
|
||||
if (error != NULL)
|
||||
*error = "host name resolution timed out";
|
||||
return 0;
|
||||
}
|
||||
|
||||
hts_mutexlock(&opt->state.lock);
|
||||
{ /* store the full list (coucal owns the record and dups the host key; a
|
||||
concurrent resolve of the same host replaces, and frees, this one) */
|
||||
t_dnscache *const record = malloct(sizeof(t_dnscache));
|
||||
|
||||
if (record != NULL) {
|
||||
memset(record, 0, sizeof(*record));
|
||||
record->host_count = count;
|
||||
@@ -5137,28 +5263,13 @@ static int hts_dns_resolve_list_(httrackp *opt, const char *_iadr,
|
||||
memcpy(record->host_addr[i], &SOCaddr_sockaddr(resolved[i]),
|
||||
record->host_length[i]);
|
||||
}
|
||||
coucal_add_pvoid(cache, iadr, record);
|
||||
coucal_add_pvoid(cache, host, record);
|
||||
}
|
||||
|
||||
/* copy result to caller (cache store may have failed; result still valid)
|
||||
*/
|
||||
for (i = 0; i < count && i < max; i++) {
|
||||
SOCaddr_copy_SOCaddr(out[i], resolved[i]);
|
||||
}
|
||||
return count;
|
||||
} // retour hp du cache
|
||||
}
|
||||
|
||||
int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
|
||||
const char **error) {
|
||||
int count;
|
||||
|
||||
if (!strnotempty(iadr) || max <= 0) {
|
||||
return 0;
|
||||
}
|
||||
hts_mutexlock(&opt->state.lock);
|
||||
count = hts_dns_resolve_list_(opt, iadr, out, max, error);
|
||||
hts_mutexrelease(&opt->state.lock);
|
||||
|
||||
/* copy result to caller (cache store may have failed; result still valid) */
|
||||
dns_copy_addrs(out, resolved, count, max);
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
@@ -222,7 +222,8 @@ LLint http_xfread1(htsblk * r, int bufl);
|
||||
/* Cached resolver: fill out[0..count-1] with up to max addresses for iadr (in
|
||||
resolver order), returning the count (0 = does not resolve, negative-cached).
|
||||
Resolves once per host; later calls read the DNS cache. Must hold no lock
|
||||
(brackets opt->state.lock itself). */
|
||||
(brackets opt->state.lock itself, never across the resolve). A miss resolves
|
||||
on a worker thread bounded by opt->timeout; a timeout reports 0, uncached. */
|
||||
int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
|
||||
const char **error);
|
||||
HTS_INLINE SOCaddr *hts_dns_resolve2(httrackp *opt, const char *iadr,
|
||||
|
||||
@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htsdefines.h"
|
||||
#include "htslib.h"
|
||||
#include "htsalias.h"
|
||||
#include "htsparse.h"
|
||||
#include "htscache.h"
|
||||
#include "htscache_selftest.h"
|
||||
@@ -825,6 +826,21 @@ static int st_simplify(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_expandhome(httrackp *opt, int argc, char **argv) {
|
||||
String path = STRING_EMPTY;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "expandhome: needs a path\n");
|
||||
return 1;
|
||||
}
|
||||
StringCopy(path, argv[0]);
|
||||
expand_home(&path);
|
||||
printf("expanded=%s\n", StringBuff(path));
|
||||
StringFree(path);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_mime(httrackp *opt, int argc, char **argv) {
|
||||
char mime[256];
|
||||
|
||||
@@ -2109,6 +2125,15 @@ static int st_dns(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_dnstimeout(httrackp *opt, int argc, char **argv) {
|
||||
const int err = dns_timeout_selftests(opt);
|
||||
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
printf("dns-timeout-selftest: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_cookies(httrackp *opt, int argc, char **argv) {
|
||||
static t_cookie cookie;
|
||||
char hdr[1024];
|
||||
@@ -3070,6 +3095,7 @@ static const struct selftest_entry {
|
||||
{"filterbounds", "", "matcher length/work caps reject hostile patterns",
|
||||
st_filterbounds},
|
||||
{"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
|
||||
{"expandhome", "<path>", "expand a leading ~/ into $HOME", st_expandhome},
|
||||
{"stripquery", "", "--strip-query pattern/key stripping self-test",
|
||||
st_stripquery},
|
||||
{"urlhack", "", "-%u url-hack sub-flag (www/slash/query) self-test",
|
||||
@@ -3133,6 +3159,8 @@ static const struct selftest_entry {
|
||||
{"cache-corrupt", "<dir>", "cache read-side corruption self-test",
|
||||
st_cache_corrupt},
|
||||
{"dns", "", "DNS resolver/cache self-test", st_dns},
|
||||
{"dnstimeout", "", "a slow DNS resolve is bounded and holds no lock",
|
||||
st_dnstimeout},
|
||||
{"cookies", "", "cookie request-header self-test", st_cookies},
|
||||
{"useragent", "", "default User-Agent self-test", st_useragent},
|
||||
{"makeindex", "[dir]", "hts_finish_makeindex footer/refresh self-test",
|
||||
|
||||
15
tests/01_engine-dnstimeout.test
Normal file
15
tests/01_engine-dnstimeout.test
Normal file
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# A resolve against a deliberately slow mock getaddrinfo (no network) must
|
||||
# return on --timeout, must not hold opt->state.lock meanwhile, and must not
|
||||
# cache the timeout as an answer (#606).
|
||||
# 'run' is an ignored placeholder argument.
|
||||
out=$(httrack -#test=dnstimeout run)
|
||||
|
||||
test "$out" = "dns-timeout-selftest: OK" || {
|
||||
echo "expected 'dns-timeout-selftest: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
53
tests/01_engine-expandhome.test
Normal file
53
tests/01_engine-expandhome.test
Normal file
@@ -0,0 +1,53 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
# SC2088: the literal, unexpanded '~' is the input under test.
|
||||
# shellcheck disable=SC2088
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ~ expansion for the -O base path (expand_home). $HOME is pinned so the cases
|
||||
# cannot depend on the caller's, but MSYS rewrites it into a Windows path
|
||||
# before the native httrack.exe reads it: ask the engine what it saw rather
|
||||
# than hardcoding the value.
|
||||
ask() {
|
||||
HOME=HTSHOME httrack -O /dev/null -#test=expandhome "$1"
|
||||
}
|
||||
exp() {
|
||||
test "$(ask "$1")" == "expanded=$2" || exit 1
|
||||
}
|
||||
|
||||
home=$(ask '~')
|
||||
home=${home#expanded=}
|
||||
|
||||
# or every case below is vacuous: '~' means expansion never fired, '.' means it
|
||||
# fell back to the default without ever reading $HOME
|
||||
test "$home" != '~' || exit 1
|
||||
test "$home" != '.' || exit 1
|
||||
|
||||
exp '~' "$home"
|
||||
exp '~/foo' "$home/foo"
|
||||
exp '~/foo/bar/#' "$home/foo/bar/#"
|
||||
exp '~/' "$home/"
|
||||
exp '~/../x' "$home/../x"
|
||||
|
||||
# ~user/ needs getpwnam: left alone rather than mangled into $HOME + "user/"
|
||||
exp '~smith/foo' '~smith/foo'
|
||||
exp '~root' '~root'
|
||||
|
||||
# only a leading '~' expands; anything else is a literal path component
|
||||
exp 'a~/x' 'a~/x'
|
||||
exp './~/x' './~/x'
|
||||
exp 'foo~bar' 'foo~bar'
|
||||
|
||||
# an unset or empty $HOME must not turn ~/foo into the absolute /foo
|
||||
test "$(env -u HOME httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
|
||||
test "$(HOME='' httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
|
||||
|
||||
# A $HOME past the 2 * HTS_URLMAXSIZE buffer is left alone: strcatbuff aborts
|
||||
# rather than truncates. Only meaningful where the engine sees the value we set:
|
||||
# MSYS rewrites HOME into a path of its own choosing, long or not.
|
||||
if test "$home" = HTSHOME; then
|
||||
long=$(printf '%04096d' 0 | tr '0' 'A')
|
||||
test "$(HOME="$long" httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=~/foo" || exit 1
|
||||
fi
|
||||
@@ -71,3 +71,23 @@ p 'http://a]b:c@host' 'name=http://a]b:c@host port=8080 host=host kind=http'
|
||||
|
||||
# a lone ':' must not underflow the backward scan
|
||||
p ':' 'name= port=8080 host= kind=http'
|
||||
|
||||
# a port is *DIGIT in 1..65535, else the scheme default: sscanf("%d") used to
|
||||
# wrap past INT_MAX and hand back a garbage port (#602)
|
||||
p 'http://host:65535' 'name=http://host port=65535 host=host kind=http'
|
||||
p 'http://host:1' 'name=http://host port=1 host=host kind=http'
|
||||
p 'http://host:65536' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:2147483648' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:4294967296' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:99999999999999' 'name=http://host port=8080 host=host kind=http'
|
||||
# discriminating: this one wrapped to a plausible 80, so range-checking the
|
||||
# sscanf result instead of rejecting the overflow still passes everything above
|
||||
p 'http://host:4294967376' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:999999999999999999999999999999' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:0' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:-1' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:80x' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host: 80' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:' 'name=http://host port=8080 host=host kind=http'
|
||||
# the fallback is the scheme's own default, not a hardcoded 8080
|
||||
p 'socks5://host:99999999999999' 'name=socks5://host port=1080 host=host kind=socks'
|
||||
|
||||
@@ -35,6 +35,7 @@ TESTS = \
|
||||
01_engine-copyopt.test \
|
||||
01_engine-crange.test \
|
||||
01_engine-dns.test \
|
||||
01_engine-dnstimeout.test \
|
||||
01_engine-doitlog.test \
|
||||
01_engine-entities.test \
|
||||
01_engine-filelist.test \
|
||||
@@ -58,6 +59,7 @@ TESTS = \
|
||||
01_engine-pause.test \
|
||||
01_engine-rcfile.test \
|
||||
01_engine-reconcile.test \
|
||||
01_engine-expandhome.test \
|
||||
01_engine-fsize.test \
|
||||
01_engine-redirect.test \
|
||||
01_engine-relative.test \
|
||||
|
||||
Reference in New Issue
Block a user