mirror of
https://github.com/xroche/httrack.git
synced 2026-06-19 16:53:18 +03:00
Compare commits
11 Commits
feature/ap
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b2d7afdaa | ||
|
|
6ee539619e | ||
|
|
fb098b27b4 | ||
|
|
5f6a3fb917 | ||
|
|
f9e676dbe3 | ||
|
|
1b440c44b5 | ||
|
|
ac6dd1a570 | ||
|
|
4549ec3695 | ||
|
|
ac56c31b24 | ||
|
|
ee6beeeb7d | ||
|
|
6788bda380 |
@@ -3838,7 +3838,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
|||||||
/* funny log for commandline users */
|
/* funny log for commandline users */
|
||||||
//if (!opt->quiet) {
|
//if (!opt->quiet) {
|
||||||
// petite animation
|
// petite animation
|
||||||
if (opt->verbosedisplay == 1) {
|
if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
|
||||||
if (back[i].status == STATUS_READY) {
|
if (back[i].status == STATUS_READY) {
|
||||||
if (back[i].r.statuscode == HTTP_OK)
|
if (back[i].r.statuscode == HTTP_OK)
|
||||||
printf("* %s%s (" LLintP " bytes) - OK" VT_CLREOL "\r",
|
printf("* %s%s (" LLintP " bytes) - OK" VT_CLREOL "\r",
|
||||||
|
|||||||
@@ -3342,7 +3342,8 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
|
|||||||
int ptr, int numero_passe) {
|
int ptr, int numero_passe) {
|
||||||
int n = back_pluggable_sockets(sback, opt);
|
int n = back_pluggable_sockets(sback, opt);
|
||||||
|
|
||||||
if (opt->savename_delayed == 2 && !opt->delayed_cached) /* cancel (always delayed) */
|
if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD &&
|
||||||
|
!opt->delayed_cached) /* cancel (always delayed) */
|
||||||
return 0;
|
return 0;
|
||||||
if (n > 0) {
|
if (n > 0) {
|
||||||
int p;
|
int p;
|
||||||
@@ -3702,7 +3703,9 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
|||||||
if (from->maxsoc > 0)
|
if (from->maxsoc > 0)
|
||||||
to->maxsoc = from->maxsoc;
|
to->maxsoc = from->maxsoc;
|
||||||
|
|
||||||
if (from->nearlink > -1)
|
/* hts_boolean/enum fields are unsigned (GCC), so a bare `> -1` unset-guard
|
||||||
|
is always false; cast to int to keep the -1 "unset" sentinel test. */
|
||||||
|
if ((int) from->nearlink > -1)
|
||||||
to->nearlink = from->nearlink;
|
to->nearlink = from->nearlink;
|
||||||
|
|
||||||
if (from->timeout > -1)
|
if (from->timeout > -1)
|
||||||
@@ -3729,10 +3732,10 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
|||||||
if (from->hostcontrol > -1)
|
if (from->hostcontrol > -1)
|
||||||
to->hostcontrol = from->hostcontrol;
|
to->hostcontrol = from->hostcontrol;
|
||||||
|
|
||||||
if (from->errpage > -1)
|
if ((int) from->errpage > -1)
|
||||||
to->errpage = from->errpage;
|
to->errpage = from->errpage;
|
||||||
|
|
||||||
if (from->parseall > -1)
|
if ((int) from->parseall > -1)
|
||||||
to->parseall = from->parseall;
|
to->parseall = from->parseall;
|
||||||
|
|
||||||
// test all: bit 8 de travel
|
// test all: bit 8 de travel
|
||||||
@@ -3844,7 +3847,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
|
|||||||
a = opt->savename_type;
|
a = opt->savename_type;
|
||||||
b = opt->savename_83;
|
b = opt->savename_83;
|
||||||
opt->savename_type = 0;
|
opt->savename_type = 0;
|
||||||
opt->savename_83 = 0;
|
opt->savename_83 = HTS_SAVENAME_83_LONG;
|
||||||
// note: adr,fil peuvent être patchés
|
// note: adr,fil peuvent être patchés
|
||||||
r =
|
r =
|
||||||
url_savename(&afs, NULL, NULL, NULL, opt, sback, cache, hashptr, ptr, numero_passe,
|
url_savename(&afs, NULL, NULL, NULL, opt, sback, cache, hashptr, ptr, numero_passe,
|
||||||
|
|||||||
@@ -612,12 +612,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
/* Terminal is a tty, may ask questions and display funny information */
|
/* Terminal is a tty, may ask questions and display funny information */
|
||||||
if (isatty(1)) {
|
if (isatty(1)) {
|
||||||
opt->quiet = 0;
|
opt->quiet = 0;
|
||||||
opt->verbosedisplay = 1;
|
opt->verbosedisplay = HTS_VERBOSE_SIMPLE;
|
||||||
}
|
}
|
||||||
/* Not a tty, no stdin input or funny output! */
|
/* Not a tty, no stdin input or funny output! */
|
||||||
else {
|
else {
|
||||||
opt->quiet = 1;
|
opt->quiet = 1;
|
||||||
opt->verbosedisplay = 0;
|
opt->verbosedisplay = HTS_VERBOSE_NONE;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@@ -953,9 +953,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
p = buff;
|
p = buff;
|
||||||
do {
|
do {
|
||||||
int insert_after_argc;
|
int insert_after_argc;
|
||||||
|
int quoted; /* "" unquotes to empty but is still a real token (#106) */
|
||||||
|
|
||||||
// read next
|
// read next
|
||||||
lastp = p;
|
lastp = p;
|
||||||
|
quoted = (p != NULL && *p == '"');
|
||||||
if (p) {
|
if (p) {
|
||||||
p = next_token(p, 1);
|
p = next_token(p, 1);
|
||||||
if (p) {
|
if (p) {
|
||||||
@@ -966,7 +968,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
|
|
||||||
/* Insert parameters BUT so that they can be in the same order */
|
/* Insert parameters BUT so that they can be in the same order */
|
||||||
if (lastp) {
|
if (lastp) {
|
||||||
if (strnotempty(lastp)) {
|
if (strnotempty(lastp) || quoted) {
|
||||||
insert_after_argc = argc - insert_after;
|
insert_after_argc = argc - insert_after;
|
||||||
cmdl_ins(lastp, insert_after_argc, (argv + insert_after), x_argvblk,
|
cmdl_ins(lastp, insert_after_argc, (argv + insert_after), x_argvblk,
|
||||||
x_argvblk_size, x_ptr);
|
x_argvblk_size, x_ptr);
|
||||||
@@ -1815,24 +1817,22 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
com++;
|
com++;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case 'L':
|
case 'L': {
|
||||||
{
|
sscanf(com + 1, "%d", (int *) &opt->savename_83);
|
||||||
sscanf(com + 1, "%d", &opt->savename_83);
|
switch (opt->savename_83) {
|
||||||
switch (opt->savename_83) {
|
case 0: // 8-3 (ISO9660 L1)
|
||||||
case 0: // 8-3 (ISO9660 L1)
|
opt->savename_83 = HTS_SAVENAME_83_DOS;
|
||||||
opt->savename_83 = 1;
|
break;
|
||||||
break;
|
case 1:
|
||||||
case 1:
|
opt->savename_83 = HTS_SAVENAME_83_LONG;
|
||||||
opt->savename_83 = 0;
|
break;
|
||||||
break;
|
default: // 2 == ISO9660 (ISO9660 L2)
|
||||||
default: // 2 == ISO9660 (ISO9660 L2)
|
opt->savename_83 = HTS_SAVENAME_83_ISO9660;
|
||||||
opt->savename_83 = 2;
|
break;
|
||||||
break;
|
|
||||||
}
|
|
||||||
while(isdigit((unsigned char) *(com + 1)))
|
|
||||||
com++;
|
|
||||||
}
|
}
|
||||||
break;
|
while (isdigit((unsigned char) *(com + 1)))
|
||||||
|
com++;
|
||||||
|
} break;
|
||||||
case 's':
|
case 's':
|
||||||
if (isdigit((unsigned char) *(com + 1))) {
|
if (isdigit((unsigned char) *(com + 1))) {
|
||||||
sscanf(com + 1, "%d", (int *) &opt->robots);
|
sscanf(com + 1, "%d", (int *) &opt->robots);
|
||||||
@@ -1989,7 +1989,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
}
|
}
|
||||||
break; // url hack
|
break; // url hack
|
||||||
case 'v':
|
case 'v':
|
||||||
opt->verbosedisplay = 2;
|
opt->verbosedisplay = HTS_VERBOSE_FULL;
|
||||||
if (isdigit((unsigned char) *(com + 1))) {
|
if (isdigit((unsigned char) *(com + 1))) {
|
||||||
sscanf(com + 1, "%d", (int *) &opt->verbosedisplay);
|
sscanf(com + 1, "%d", (int *) &opt->verbosedisplay);
|
||||||
while(isdigit((unsigned char) *(com + 1)))
|
while(isdigit((unsigned char) *(com + 1)))
|
||||||
@@ -2004,7 +2004,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case 'N':
|
case 'N':
|
||||||
opt->savename_delayed = 2;
|
opt->savename_delayed = HTS_SAVENAME_DELAYED_HARD;
|
||||||
if (isdigit((unsigned char) *(com + 1))) {
|
if (isdigit((unsigned char) *(com + 1))) {
|
||||||
sscanf(com + 1, "%d", (int *) &opt->savename_delayed);
|
sscanf(com + 1, "%d", (int *) &opt->savename_delayed);
|
||||||
while(isdigit((unsigned char) *(com + 1)))
|
while(isdigit((unsigned char) *(com + 1)))
|
||||||
@@ -3096,6 +3096,78 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
|||||||
htsmain_free();
|
htsmain_free();
|
||||||
return 0;
|
return 0;
|
||||||
break;
|
break;
|
||||||
|
case '9': { // copy_htsopt selftest: httrack -#9
|
||||||
|
httrackp *from = hts_create_opt();
|
||||||
|
httrackp *to = hts_create_opt();
|
||||||
|
int err = 0;
|
||||||
|
|
||||||
|
/* from-values differ from both the to-values and the
|
||||||
|
hts_create_opt() defaults (nearlink FALSE, errpage/parseall
|
||||||
|
TRUE), so a copy that no-ops or just resets to defaults is
|
||||||
|
caught too, not only the unsigned-guard bug. */
|
||||||
|
from->retry = 7; /* int field: positive control */
|
||||||
|
to->retry = 0;
|
||||||
|
from->nearlink = HTS_TRUE;
|
||||||
|
to->nearlink = HTS_FALSE;
|
||||||
|
from->errpage = HTS_FALSE;
|
||||||
|
to->errpage = HTS_TRUE;
|
||||||
|
from->parseall = HTS_FALSE;
|
||||||
|
to->parseall = HTS_TRUE;
|
||||||
|
|
||||||
|
copy_htsopt(from, to);
|
||||||
|
|
||||||
|
if (to->retry != 7)
|
||||||
|
err = 1;
|
||||||
|
if (to->nearlink != HTS_TRUE)
|
||||||
|
err = 1;
|
||||||
|
if (to->errpage != HTS_FALSE)
|
||||||
|
err = 1;
|
||||||
|
if (to->parseall != HTS_FALSE)
|
||||||
|
err = 1;
|
||||||
|
|
||||||
|
hts_free_opt(from);
|
||||||
|
hts_free_opt(to);
|
||||||
|
printf("copy-htsopt: %s\n", err ? "FAIL" : "OK");
|
||||||
|
htsmain_free();
|
||||||
|
return err;
|
||||||
|
} break;
|
||||||
|
case 'Q': { // cookie request-header selftest: httrack -#Q
|
||||||
|
static t_cookie cookie;
|
||||||
|
char hdr[1024];
|
||||||
|
/* RFC 6265: bare name=value pairs, no $Version/$Path (#151). */
|
||||||
|
const char *expected = "Cookie: name=value; has_js=1" H_CRLF;
|
||||||
|
int err = 0;
|
||||||
|
|
||||||
|
const char *dom = "www.example.com";
|
||||||
|
int added;
|
||||||
|
|
||||||
|
cookie.max_len = (int) sizeof(cookie.data);
|
||||||
|
cookie.data[0] = '\0';
|
||||||
|
added = cookie_add(&cookie, "name", "value", dom, "/");
|
||||||
|
added |= cookie_add(&cookie, "has_js", "1", dom, "/");
|
||||||
|
/* different domain: must be filtered out */
|
||||||
|
added |= cookie_add(&cookie, "junk", "x", "other.org", "/");
|
||||||
|
if (added) {
|
||||||
|
printf("cookie-header: FAIL (cookie_add setup)\n");
|
||||||
|
htsmain_free();
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
http_cookie_header_selftest(&cookie, dom, "/", hdr,
|
||||||
|
sizeof(hdr));
|
||||||
|
if (strcmp(hdr, expected) != 0)
|
||||||
|
err = 1;
|
||||||
|
if (strstr(hdr, "$Version") != NULL ||
|
||||||
|
strstr(hdr, "$Path") != NULL)
|
||||||
|
err = 1;
|
||||||
|
if (strstr(hdr, "junk") != NULL) // wrong-domain cookie leaked
|
||||||
|
err = 1;
|
||||||
|
printf("cookie-header: %s\n", err ? "FAIL" : "OK");
|
||||||
|
if (err)
|
||||||
|
printf(" got: %s\n", hdr);
|
||||||
|
htsmain_free();
|
||||||
|
return err;
|
||||||
|
} break;
|
||||||
case '!':
|
case '!':
|
||||||
HTS_PANIC_PRINTF
|
HTS_PANIC_PRINTF
|
||||||
("Option #! is disabled for security reasons");
|
("Option #! is disabled for security reasons");
|
||||||
|
|||||||
80
src/htslib.c
80
src/htslib.c
@@ -874,6 +874,50 @@ static void print_buffer(buff_struct*const str, const char *format, ...) {
|
|||||||
assertf(str->pos < str->capacity);
|
assertf(str->pos < str->capacity);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Append the request "Cookie:" header line for every stored cookie matching
|
||||||
|
domain/path. RFC 6265 form: bare "name=value" pairs joined by "; ", no
|
||||||
|
$Version/$Path attributes (those are RFC 2965 syntax that modern servers
|
||||||
|
reject, issue #151). Returns the number of cookies emitted. */
|
||||||
|
static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
|
||||||
|
const char *domain, const char *path) {
|
||||||
|
char buffer[8192];
|
||||||
|
char *b;
|
||||||
|
int cook = 0;
|
||||||
|
int max_cookies = 8;
|
||||||
|
|
||||||
|
if (cookie == NULL)
|
||||||
|
return 0;
|
||||||
|
b = cookie->data;
|
||||||
|
do {
|
||||||
|
b = cookie_find(b, "", domain, path); // next matching cookie
|
||||||
|
if (b != NULL) {
|
||||||
|
max_cookies--;
|
||||||
|
if (!cook) {
|
||||||
|
print_buffer(bstr, "Cookie: ");
|
||||||
|
cook = 1;
|
||||||
|
} else
|
||||||
|
print_buffer(bstr, "; ");
|
||||||
|
print_buffer(bstr, "%s", cookie_get(buffer, b, 5));
|
||||||
|
print_buffer(bstr, "=%s", cookie_get(buffer, b, 6));
|
||||||
|
b = cookie_nextfield(b);
|
||||||
|
}
|
||||||
|
} while (b != NULL && max_cookies > 0);
|
||||||
|
if (cook)
|
||||||
|
print_buffer(bstr, H_CRLF);
|
||||||
|
return cook;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Self-test entry for append_cookie_header(): build the request Cookie line
|
||||||
|
into dst (always NUL-terminated). Returns the number of cookies emitted. */
|
||||||
|
int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
|
||||||
|
const char *path, char *dst, size_t dst_size) {
|
||||||
|
buff_struct bstr = {dst, dst_size, 0};
|
||||||
|
|
||||||
|
assertf(dst != NULL && dst_size > 0);
|
||||||
|
dst[0] = '\0';
|
||||||
|
return append_cookie_header(&bstr, cookie, domain, path);
|
||||||
|
}
|
||||||
|
|
||||||
// envoi d'une requète
|
// envoi d'une requète
|
||||||
int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||||
const char *xsend, const char *adr, const char *fil,
|
const char *xsend, const char *adr, const char *fil,
|
||||||
@@ -1048,34 +1092,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
|||||||
search_tag + strlen(POSTTOK) + 1))));
|
search_tag + strlen(POSTTOK) + 1))));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// gestion cookies?
|
// send stored cookies matching this host/path
|
||||||
if (cookie) {
|
if (cookie) {
|
||||||
char buffer[8192];
|
append_cookie_header(&bstr, cookie, jump_identification_const(adr), fil);
|
||||||
char *b = cookie->data;
|
|
||||||
int cook = 0;
|
|
||||||
int max_cookies = 8;
|
|
||||||
|
|
||||||
do {
|
|
||||||
b = cookie_find(b, "", jump_identification_const(adr), fil); // prochain cookie satisfaisant aux conditions
|
|
||||||
if (b != NULL) {
|
|
||||||
max_cookies--;
|
|
||||||
if (!cook) {
|
|
||||||
print_buffer(&bstr, "Cookie: $Version=1; ");
|
|
||||||
cook = 1;
|
|
||||||
} else
|
|
||||||
print_buffer(&bstr, "; ");
|
|
||||||
print_buffer(&bstr, "%s", cookie_get(buffer, b, 5));
|
|
||||||
print_buffer(&bstr, "=%s", cookie_get(buffer, b, 6));
|
|
||||||
print_buffer(&bstr, "; $Path=%s", cookie_get(buffer, b, 2));
|
|
||||||
b = cookie_nextfield(b);
|
|
||||||
}
|
|
||||||
} while(b != NULL && max_cookies > 0);
|
|
||||||
if (cook) { // on a envoyé un (ou plusieurs) cookie?
|
|
||||||
print_buffer(&bstr, H_CRLF);
|
|
||||||
#if DEBUG_COOK
|
|
||||||
printf("Header:\n%s\n", bstr.buffer);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// gérer le keep-alive (garder socket)
|
// gérer le keep-alive (garder socket)
|
||||||
if (retour->req.http11 && !retour->req.nokeepalive) {
|
if (retour->req.http11 && !retour->req.nokeepalive) {
|
||||||
@@ -5468,9 +5487,10 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
|||||||
"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)");
|
"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)");
|
||||||
StringCopy(opt->referer, "");
|
StringCopy(opt->referer, "");
|
||||||
StringCopy(opt->from, "");
|
StringCopy(opt->from, "");
|
||||||
opt->savename_83 = 0; // noms longs par défaut
|
opt->savename_83 = HTS_SAVENAME_83_LONG; // long names by default
|
||||||
opt->savename_type = 0; // avec structure originale
|
opt->savename_type = 0; // avec structure originale
|
||||||
opt->savename_delayed = 2; // hard delayed type (default)
|
opt->savename_delayed =
|
||||||
|
HTS_SAVENAME_DELAYED_HARD; // always delay the type check (default)
|
||||||
opt->delayed_cached = HTS_TRUE;
|
opt->delayed_cached = HTS_TRUE;
|
||||||
opt->mimehtml = HTS_FALSE;
|
opt->mimehtml = HTS_FALSE;
|
||||||
opt->parsejava = HTSPARSE_DEFAULT; // parser classes
|
opt->parsejava = HTSPARSE_DEFAULT; // parser classes
|
||||||
@@ -5495,7 +5515,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
|||||||
opt->parseall = HTS_TRUE;
|
opt->parseall = HTS_TRUE;
|
||||||
opt->parsedebug = HTS_FALSE;
|
opt->parsedebug = HTS_FALSE;
|
||||||
opt->norecatch = HTS_FALSE;
|
opt->norecatch = HTS_FALSE;
|
||||||
opt->verbosedisplay = 0; // pas d'animation texte
|
opt->verbosedisplay = HTS_VERBOSE_NONE; // no text animation
|
||||||
opt->sizehack = HTS_FALSE;
|
opt->sizehack = HTS_FALSE;
|
||||||
opt->urlhack = HTS_TRUE;
|
opt->urlhack = HTS_TRUE;
|
||||||
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
|
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
|
||||||
|
|||||||
@@ -182,6 +182,11 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
|
|||||||
const char *adr, const char *fil,
|
const char *adr, const char *fil,
|
||||||
const char *referer_adr, const char *referer_fil,
|
const char *referer_adr, const char *referer_fil,
|
||||||
htsblk * retour);
|
htsblk * retour);
|
||||||
|
/* Build the request "Cookie:" header line for stored cookies matching
|
||||||
|
domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
|
||||||
|
wraps the same logic http_sendhead() uses. Returns cookies emitted. */
|
||||||
|
int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
|
||||||
|
const char *path, char *dst, size_t dst_size);
|
||||||
|
|
||||||
//int newhttp(char* iadr,char* err=NULL);
|
//int newhttp(char* iadr,char* err=NULL);
|
||||||
T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
|
T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
|
||||||
|
|||||||
@@ -184,10 +184,11 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
|
|
||||||
/* 8-3 ? */
|
/* 8-3 ? */
|
||||||
switch (opt->savename_83) {
|
switch (opt->savename_83) {
|
||||||
case 1: // 8-3
|
case HTS_SAVENAME_83_DOS: // 8-3
|
||||||
max_char = 8;
|
max_char = 8;
|
||||||
break;
|
break;
|
||||||
case 2: // Level 2 File names may be up to 31 characters.
|
case HTS_SAVENAME_83_ISO9660: // Level 2 File names may be up to 31
|
||||||
|
// characters.
|
||||||
max_char = 31;
|
max_char = 31;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
@@ -324,7 +325,7 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* replace shtml to html.. */
|
/* replace shtml to html.. */
|
||||||
if (opt->savename_delayed == 2)
|
if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD)
|
||||||
is_html = -1; /* ALWAYS delay type */
|
is_html = -1; /* ALWAYS delay type */
|
||||||
else
|
else
|
||||||
is_html = ishtml(opt, fil);
|
is_html = ishtml(opt, fil);
|
||||||
@@ -363,7 +364,9 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
) {
|
) {
|
||||||
// tester type avec requète HEAD si on ne connait pas le type du fichier
|
// tester type avec requète HEAD si on ne connait pas le type du fichier
|
||||||
if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/'))) // slash doit être html?
|
if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/'))) // slash doit être html?
|
||||||
if (opt->savename_delayed == 2 || (ishtest = ishtml(opt, fil)) < 0) { // on ne sait pas si c'est un html ou un fichier..
|
if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
|
||||||
|
(ishtest = ishtml(opt, fil)) <
|
||||||
|
0) { // unsure whether it's html or a file
|
||||||
// lire dans le cache
|
// lire dans le cache
|
||||||
htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
|
htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
|
||||||
|
|
||||||
@@ -393,11 +396,12 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
//
|
//
|
||||||
} else if (opt->savename_delayed != 2 && is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
|
} else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD &&
|
||||||
Lookup mimetype not only by extension,
|
is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
|
||||||
but also by filename */
|
Lookup mimetype not only by extension,
|
||||||
/* Note: "foo.cgi => text/html" means that foo.cgi shall have the text/html MIME file type,
|
but also by filename */
|
||||||
that is, ".html" */
|
/* Note: "foo.cgi => text/html" means that foo.cgi shall have the
|
||||||
|
text/html MIME file type, that is, ".html" */
|
||||||
char BIGSTK mime[1024];
|
char BIGSTK mime[1024];
|
||||||
|
|
||||||
mime[0] = ext[0] = '\0';
|
mime[0] = ext[0] = '\0';
|
||||||
@@ -408,9 +412,13 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// note: if savename_delayed is enabled, the naming will be temporary (and slightly invalid!)
|
// note: if savename_delayed is enabled, the naming will be temporary
|
||||||
// note: if we are about to stop (opt->state.stop), back_add() will fail later
|
// (and slightly invalid!)
|
||||||
else if (opt->savename_delayed != 0 && !opt->state.stop) {
|
//
|
||||||
|
// note: if we are about to stop (opt->state.stop), back_add() will
|
||||||
|
// fail later
|
||||||
|
else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
|
||||||
|
!opt->state.stop) {
|
||||||
// Check if the file is ready in backing. We basically take the same logic as later.
|
// Check if the file is ready in backing. We basically take the same logic as later.
|
||||||
// FIXME: we should cleanup and factorize this unholy mess
|
// FIXME: we should cleanup and factorize this unholy mess
|
||||||
if (headers != NULL && headers->status >= 0 && !is_redirect) {
|
if (headers != NULL && headers->status >= 0 && !is_redirect) {
|
||||||
@@ -698,7 +706,7 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
}
|
}
|
||||||
// restaurer
|
// restaurer
|
||||||
opt->state._hts_in_html_parsing = hihp;
|
opt->state._hts_in_html_parsing = hihp;
|
||||||
} // caché?
|
} // caché?
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1190,7 +1198,8 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
// Not used anymore unless non-delayed types.
|
// Not used anymore unless non-delayed types.
|
||||||
// de même en cas de manque d'extension on en place une de manière forcée..
|
// de même en cas de manque d'extension on en place une de manière forcée..
|
||||||
// cela évite les /chez/toto et les /chez/toto/index.html incompatibles
|
// cela évite les /chez/toto et les /chez/toto/index.html incompatibles
|
||||||
if (opt->savename_type != -1 && opt->savename_delayed != 2) {
|
if (opt->savename_type != -1 &&
|
||||||
|
opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD) {
|
||||||
char *a = afs->save + strlen(afs->save) - 1;
|
char *a = afs->save + strlen(afs->save) - 1;
|
||||||
|
|
||||||
while((a > afs->save) && (*a != '.') && (*a != '/'))
|
while((a > afs->save) && (*a != '.') && (*a != '/'))
|
||||||
@@ -1236,31 +1245,21 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
size_t i;
|
size_t i;
|
||||||
for(i = 0 ; afs->save[i] != '\0' ; i++) {
|
for(i = 0 ; afs->save[i] != '\0' ; i++) {
|
||||||
unsigned char c = (unsigned char) afs->save[i];
|
unsigned char c = (unsigned char) afs->save[i];
|
||||||
if (c < 32 // control
|
if (c < 32 // control
|
||||||
|| c == 127 // unwise
|
|| c == 127 // unwise
|
||||||
|| c == '~' // unix unwise
|
|| c == '~' // unix unwise
|
||||||
|| c == '\\' // windows separator
|
|| c == '\\' // windows separator
|
||||||
|| c == ':' // windows forbidden
|
|| c == ':' // windows forbidden
|
||||||
|| c == '*' // windows forbidden
|
|| c == '*' // windows forbidden
|
||||||
|| c == '?' // windows forbidden
|
|| c == '?' // windows forbidden
|
||||||
|| c == '\"' // windows forbidden
|
|| c == '\"' // windows forbidden
|
||||||
|| c == '<' // windows forbidden
|
|| c == '<' // windows forbidden
|
||||||
|| c == '>' // windows forbidden
|
|| c == '>' // windows forbidden
|
||||||
|| c == '|' // windows forbidden
|
|| c == '|' // windows forbidden
|
||||||
//|| c == '@' // ?
|
//|| c == '@' // ?
|
||||||
||
|
|| (opt->savename_83 == HTS_SAVENAME_83_ISO9660 // CDROM
|
||||||
(
|
&& (c == '-' || c == '=' || c == '+'))) {
|
||||||
opt->savename_83 == 2 // CDROM
|
afs->save[i] = '_';
|
||||||
&&
|
|
||||||
(
|
|
||||||
c == '-'
|
|
||||||
|| c == '='
|
|
||||||
|| c == '+'
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
{
|
|
||||||
afs->save[i] = '_';
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1521,7 +1520,8 @@ int url_savename(lien_adrfilsave *const afs,
|
|||||||
char *a = afs->save + strlen(afs->save) - 1;
|
char *a = afs->save + strlen(afs->save) - 1;
|
||||||
char *b;
|
char *b;
|
||||||
int n = 2;
|
int n = 2;
|
||||||
char collisionSeparator = ((opt->savename_83 != 2) ? '-' : '_');
|
char collisionSeparator =
|
||||||
|
((opt->savename_83 != HTS_SAVENAME_83_ISO9660) ? '-' : '_');
|
||||||
|
|
||||||
tempo[0] = '\0';
|
tempo[0] = '\0';
|
||||||
|
|
||||||
|
|||||||
20
src/htsopt.h
20
src/htsopt.h
@@ -342,17 +342,17 @@ typedef enum hts_seeker {
|
|||||||
HTS_SEEKER_UP = 1 << 1 /**< may ascend to parent directories */
|
HTS_SEEKER_UP = 1 << 1 /**< may ascend to parent directories */
|
||||||
} hts_seeker;
|
} hts_seeker;
|
||||||
|
|
||||||
/* Link-following scope, stored in the low byte of opt->travel. */
|
/* opt->travel: link-following scope in the low byte, flags OR'd in above it. */
|
||||||
typedef enum hts_travel_scope {
|
typedef enum hts_travel_scope {
|
||||||
HTS_TRAVEL_SAME_ADDRESS = 0, /**< stay on the same address (host) */
|
HTS_TRAVEL_SAME_ADDRESS = 0, /**< stay on the same address (host) */
|
||||||
HTS_TRAVEL_SAME_DOMAIN = 1, /**< stay on the same principal domain */
|
HTS_TRAVEL_SAME_DOMAIN = 1, /**< stay on the same principal domain */
|
||||||
HTS_TRAVEL_SAME_TLD = 2, /**< stay on the same TLD (e.g. .com) */
|
HTS_TRAVEL_SAME_TLD = 2, /**< stay on the same TLD (e.g. .com) */
|
||||||
HTS_TRAVEL_EVERYWHERE = 7 /**< follow links anywhere on the web */
|
HTS_TRAVEL_EVERYWHERE = 7, /**< follow links anywhere on the web */
|
||||||
|
HTS_TRAVEL_TEST_ALL = 1 << 8 /**< also test forbidden URLs (-t) */
|
||||||
} hts_travel_scope;
|
} hts_travel_scope;
|
||||||
|
|
||||||
/* Flags OR'd into opt->travel above the scope value. */
|
/* Mask selecting the scope value out of opt->travel. */
|
||||||
#define HTS_TRAVEL_SCOPE_MASK 0xff /**< mask selecting the scope value */
|
#define HTS_TRAVEL_SCOPE_MASK 0xff
|
||||||
#define HTS_TRAVEL_TEST_ALL (1 << 8) /**< also test forbidden URLs (-t) */
|
|
||||||
|
|
||||||
/* Text progress display detail (opt->verbosedisplay). */
|
/* Text progress display detail (opt->verbosedisplay). */
|
||||||
typedef enum hts_verbosedisplay {
|
typedef enum hts_verbosedisplay {
|
||||||
@@ -368,6 +368,13 @@ typedef enum hts_savename_delayed {
|
|||||||
HTS_SAVENAME_DELAYED_HARD = 2 /**< always delay the type check (default) */
|
HTS_SAVENAME_DELAYED_HARD = 2 /**< always delay the type check (default) */
|
||||||
} hts_savename_delayed;
|
} hts_savename_delayed;
|
||||||
|
|
||||||
|
/* Saved-name length layout (opt->savename_83). */
|
||||||
|
typedef enum hts_savename_83 {
|
||||||
|
HTS_SAVENAME_83_LONG = 0, /**< long file names (default) */
|
||||||
|
HTS_SAVENAME_83_DOS = 1, /**< DOS 8.3 names (ISO9660 level 1) */
|
||||||
|
HTS_SAVENAME_83_ISO9660 = 2 /**< ISO9660 level 2 names (up to 31 chars) */
|
||||||
|
} hts_savename_83;
|
||||||
|
|
||||||
/* Host-banning triggers (opt->hostcontrol bitmask). */
|
/* Host-banning triggers (opt->hostcontrol bitmask). */
|
||||||
typedef enum hts_hostcontrol {
|
typedef enum hts_hostcontrol {
|
||||||
HTS_HOSTCONTROL_BAN_TIMEOUT = 1 << 0, /**< ban a timing-out host */
|
HTS_HOSTCONTROL_BAN_TIMEOUT = 1 << 0, /**< ban a timing-out host */
|
||||||
@@ -430,7 +437,8 @@ struct httrackp {
|
|||||||
// int aff_progress; // progress bar
|
// int aff_progress; // progress bar
|
||||||
hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
|
hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
|
||||||
t_proxy proxy; /**< proxy configuration */
|
t_proxy proxy; /**< proxy configuration */
|
||||||
int savename_83; /**< force 8.3 (DOS) file names */
|
hts_savename_83
|
||||||
|
savename_83; /**< saved-name length layout (long/DOS/ISO9660) */
|
||||||
int savename_type; /**< saved-name layout (original tree, flat, ...) */
|
int savename_type; /**< saved-name layout (original tree, flat, ...) */
|
||||||
String
|
String
|
||||||
savename_userdef; /**< user-defined name template (e.g. %h%p/%n%q.%t) */
|
savename_userdef; /**< user-defined name template (e.g. %h%p/%n%q.%t) */
|
||||||
|
|||||||
@@ -4262,10 +4262,10 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
|
|||||||
char com[256];
|
char com[256];
|
||||||
|
|
||||||
linput(stdin, com, 200);
|
linput(stdin, com, 200);
|
||||||
if (opt->verbosedisplay == 2)
|
if (opt->verbosedisplay == HTS_VERBOSE_FULL)
|
||||||
opt->verbosedisplay = 1;
|
opt->verbosedisplay = HTS_VERBOSE_SIMPLE;
|
||||||
else
|
else
|
||||||
opt->verbosedisplay = 2;
|
opt->verbosedisplay = HTS_VERBOSE_FULL;
|
||||||
/* Info for wrappers */
|
/* Info for wrappers */
|
||||||
hts_log_print(opt, LOG_INFO, "engine: change-options");
|
hts_log_print(opt, LOG_INFO, "engine: change-options");
|
||||||
RUN_CALLBACK0(opt, chopt);
|
RUN_CALLBACK0(opt, chopt);
|
||||||
@@ -4375,7 +4375,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
|
|||||||
printf("%c\x0d", ("/-\\|")[roll]);
|
printf("%c\x0d", ("/-\\|")[roll]);
|
||||||
fflush(stdout);
|
fflush(stdout);
|
||||||
}
|
}
|
||||||
} else if (opt->verbosedisplay == 1) {
|
} else if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
|
||||||
if (b >= 0) {
|
if (b >= 0) {
|
||||||
if (back[b].r.statuscode == HTTP_OK)
|
if (back[b].r.statuscode == HTTP_OK)
|
||||||
printf("%d/%d: %s%s (" LLintP " bytes) - OK\33[K\r", ptr, opt->lien_tot,
|
printf("%d/%d: %s%s (" LLintP " bytes) - OK\33[K\r", ptr, opt->lien_tot,
|
||||||
@@ -4466,8 +4466,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
|||||||
char in_error_msg[32];
|
char in_error_msg[32];
|
||||||
|
|
||||||
// resolve unresolved type
|
// resolve unresolved type
|
||||||
if (opt->savename_delayed != 0 && *forbidden_url == 0 && IS_DELAYED_EXT(afs->save)
|
if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
|
||||||
&& !opt->state.stop) {
|
*forbidden_url == 0 && IS_DELAYED_EXT(afs->save) && !opt->state.stop) {
|
||||||
int loops;
|
int loops;
|
||||||
int continue_loop;
|
int continue_loop;
|
||||||
|
|
||||||
@@ -4851,7 +4851,7 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // delayed type check ?
|
} // delayed type check ?
|
||||||
|
|
||||||
ENGINE_SAVE_CONTEXT_BASE();
|
ENGINE_SAVE_CONTEXT_BASE();
|
||||||
|
|
||||||
|
|||||||
@@ -288,7 +288,7 @@ static void __cdecl htsshow_uninit(t_hts_callbackarg * carg) {
|
|||||||
}
|
}
|
||||||
static int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt) {
|
static int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt) {
|
||||||
use_show = 0;
|
use_show = 0;
|
||||||
if (opt->verbosedisplay == 2) {
|
if (opt->verbosedisplay == HTS_VERBOSE_FULL) {
|
||||||
use_show = 1;
|
use_show = 1;
|
||||||
vt_clear();
|
vt_clear();
|
||||||
}
|
}
|
||||||
@@ -852,7 +852,7 @@ static void sig_doback(int blind) { // mettre en backing
|
|||||||
if (global_opt != NULL) {
|
if (global_opt != NULL) {
|
||||||
// suppress logging and asking lousy questions
|
// suppress logging and asking lousy questions
|
||||||
global_opt->quiet = 1;
|
global_opt->quiet = 1;
|
||||||
global_opt->verbosedisplay = 0;
|
global_opt->verbosedisplay = HTS_VERBOSE_NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!blind)
|
if (!blind)
|
||||||
|
|||||||
15
tests/01_engine-cookies.test
Executable file
15
tests/01_engine-cookies.test
Executable file
@@ -0,0 +1,15 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Issue #151 guard: the request Cookie header must be bare RFC 6265 name=value
|
||||||
|
# pairs, no $Version/$Path attributes. Driven by the 'httrack -#Q' selftest.
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# A trailing token is required; a bare '-#Q' falls through to the usage screen.
|
||||||
|
out=$(httrack -#Q run)
|
||||||
|
|
||||||
|
# Exact-match the success line so a fall-through to usage can't pass the test.
|
||||||
|
test "$out" = "cookie-header: OK" || {
|
||||||
|
echo "expected 'cookie-header: OK', got: $out" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
17
tests/01_engine-copyopt.test
Executable file
17
tests/01_engine-copyopt.test
Executable file
@@ -0,0 +1,17 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Regression guard for the unsigned-enum sentinel trap: copy_htsopt's
|
||||||
|
# `if (from->X > -1)` guard is always false for unsigned hts_boolean fields, so
|
||||||
|
# they silently stop being copied. Driven by the in-process 'httrack -#9' test.
|
||||||
|
# Keep POSIX-portable (harness runs it via $(BASH), a plain /bin/sh on macOS).
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# A trailing token is required; a bare '-#9' falls through to the usage screen.
|
||||||
|
out=$(httrack -#9 run)
|
||||||
|
|
||||||
|
# Exact-match the success line so a fall-through to usage can't pass the test.
|
||||||
|
test "$out" = "copy-htsopt: OK" || {
|
||||||
|
echo "expected 'copy-htsopt: OK', got: $out" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
@@ -89,4 +89,37 @@ grep -q NEWCONTENT "$(find "$out" -path '*/a.html' -print -quit)" || {
|
|||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# --- 3. an empty quoted arg survives the doit.log round-trip (#106) ----------
|
||||||
|
# -%F "" (empty footer) records an empty "" token in doit.log; -r2 follows it so
|
||||||
|
# a "drop the empty token" bug shifts -r2 into -%F's slot (the reprise then sees
|
||||||
|
# -%F -r2 and panics "%F needs to be followed by ..."), making the bug visible
|
||||||
|
# rather than a harmless run off the end of argv.
|
||||||
|
out2="$tmp/out2"
|
||||||
|
rc=0
|
||||||
|
"$bin" "$url" -O "$out2" --quiet -n -%v0 -%F "" -r2 >/dev/null 2>&1 || rc=$?
|
||||||
|
test "$rc" -eq 0 || {
|
||||||
|
echo "FAIL: initial mirror with empty footer exited $rc"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
# precondition: the writer put the empty token on disk for the reader to reload.
|
||||||
|
grep -q ' -%F "" -r2' "$out2/hts-cache/doit.log" || {
|
||||||
|
echo "FAIL: empty footer not recorded as -%F \"\" -r2 in doit.log"
|
||||||
|
grep -- '-%F' "$out2/hts-cache/doit.log" || true
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
# no-url reprise: the reader rebuilds argv from doit.log and rewrites doit.log
|
||||||
|
# from it. The empty token surviving in the regenerated file proves the reader
|
||||||
|
# kept it (a drop/swallow would panic above or rewrite -%F without the "").
|
||||||
|
rc=0
|
||||||
|
"$bin" -O "$out2" --quiet >/dev/null 2>&1 || rc=$?
|
||||||
|
test "$rc" -eq 0 || {
|
||||||
|
echo "FAIL: empty-footer reprise exited $rc (empty token dropped from doit.log?)"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
grep -q ' -%F "" -r2' "$out2/hts-cache/doit.log" || {
|
||||||
|
echo "FAIL: empty footer did not survive the doit.log reload round-trip"
|
||||||
|
grep -- '-%F' "$out2/hts-cache/doit.log" || true
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ TESTS = \
|
|||||||
01_engine-cache-golden.test \
|
01_engine-cache-golden.test \
|
||||||
01_engine-charset.test \
|
01_engine-charset.test \
|
||||||
01_engine-cmdline.test \
|
01_engine-cmdline.test \
|
||||||
|
01_engine-cookies.test \
|
||||||
|
01_engine-copyopt.test \
|
||||||
01_engine-doitlog.test \
|
01_engine-doitlog.test \
|
||||||
01_engine-entities.test \
|
01_engine-entities.test \
|
||||||
01_engine-filter.test \
|
01_engine-filter.test \
|
||||||
|
|||||||
Reference in New Issue
Block a user