mirror of
https://github.com/xroche/httrack.git
synced 2026-07-06 17:07:18 +03:00
Compare commits
10 Commits
mkdeb-lint
...
remove-dea
| Author | SHA1 | Date | |
|---|---|---|---|
|
cf483a24e4 | ||
|
|
abaea566bd | ||
|
|
f66ec195aa | ||
|
|
7928db5729 | ||
|
|
20be48d56c | ||
|
|
484fc47eab | ||
|
|
abaf9b69a2 | ||
|
|
b0466b1d7b | ||
|
|
f785286c87 | ||
|
|
440a8603a9 |
9
debian/changelog
vendored
9
debian/changelog
vendored
@@ -10,6 +10,15 @@ httrack (3.49.11-1) unstable; urgency=medium
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 05 Jul 2026 00:03:18 +0200
|
||||
|
||||
httrack (3.49.10-2) unstable; urgency=medium
|
||||
|
||||
* Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
|
||||
absent (the local-server tests need python3, which the buildds lack). Add
|
||||
patches/skip-local-pause-test-without-python3.patch to guard the test on
|
||||
python3 up front, like its siblings, so it skips cleanly.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 28 Jun 2026 20:18:46 +0200
|
||||
|
||||
httrack (3.49.10-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: new download-pacing and URL-handling options plus a
|
||||
|
||||
@@ -283,7 +283,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
|
||||
char *position;
|
||||
int need_param = 1;
|
||||
|
||||
//int return_param=0;
|
||||
int pos;
|
||||
|
||||
command[0] = param[0] = addcommand[0] = '\0';
|
||||
@@ -361,7 +360,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
|
||||
strlcatbuff(return_argv[0], "0", return_argv_size);
|
||||
else if (strcmp(param, "on") == 0) {
|
||||
// on is the default
|
||||
// strcatbuff(return_argv[0],"1");
|
||||
} else
|
||||
strlcatbuff(return_argv[0], param, return_argv_size);
|
||||
}
|
||||
|
||||
153
src/htsback.c
153
src/htsback.c
@@ -43,14 +43,12 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htsback.h"
|
||||
|
||||
//#ifdef _WIN32
|
||||
#include "htsftp.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#else
|
||||
#error HTS_USEZLIB not defined
|
||||
#endif
|
||||
//#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
#ifndef __cplusplus
|
||||
@@ -572,9 +570,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
&& back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
|
||||
if (back[p].status == STATUS_READY) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"file not stored in cache due to bogus state (broken size, expected "
|
||||
LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
|
||||
back[p].r.size, back[p].url_adr, back[p].url_fil);
|
||||
"incomplete transfer (expected " LLintP
|
||||
" bytes, got " LLintP
|
||||
"): file not cached, will be retried on the next update"
|
||||
" (use -%%B to cache anyway): %s%s",
|
||||
back[p].r.totalsize, back[p].r.size, back[p].url_adr,
|
||||
back[p].url_fil);
|
||||
} else {
|
||||
hts_log_print(opt, LOG_INFO,
|
||||
"incomplete file not yet stored in cache (expected "
|
||||
@@ -594,7 +595,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
#if HTS_USEZLIB
|
||||
if (back[p].r.compressed) {
|
||||
if (back[p].r.size > 0) {
|
||||
//if ( (back[p].r.adr) && (back[p].r.size>0) ) {
|
||||
// stats
|
||||
back[p].compressed_size = back[p].r.size;
|
||||
// en mémoire -> passage sur disque
|
||||
@@ -879,11 +879,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
back[p].url_fil, NULL);
|
||||
} else {
|
||||
/* Partial file, but marked as "ok" ? */
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"file not stored in cache due to bogus state (incomplete type with %s (%d), size "
|
||||
LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
|
||||
(LLint) back[p].r.size, back[p].url_adr,
|
||||
back[p].url_fil);
|
||||
hts_log_print(
|
||||
opt, LOG_WARNING,
|
||||
"file with unresolved type not cached (%s (%d), size " LLintP
|
||||
"): %s%s",
|
||||
back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
|
||||
back[p].url_adr, back[p].url_fil);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -923,7 +924,6 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
/* clear everything but connection: switch, close, and reswitch */
|
||||
back_connxfr(src, &tmp);
|
||||
back_delete(opt, cache, sback, p);
|
||||
//deletehttp(src);
|
||||
back_connxfr(&tmp, src);
|
||||
src->req.flush_garbage = 1; /* ignore CRLF garbage */
|
||||
return 1;
|
||||
@@ -1358,9 +1358,39 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Move a still-writing .delayed placeholder to its final name (#483). */
|
||||
hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
|
||||
const char *newname) {
|
||||
hts_boolean renamed;
|
||||
|
||||
if (!back->r.is_write || back->tmpfile != NULL ||
|
||||
!IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
|
||||
return HTS_TRUE; /* nothing bound to the placeholder name */
|
||||
if (back->r.out != NULL) {
|
||||
fclose(back->r.out);
|
||||
back->r.out = NULL;
|
||||
}
|
||||
renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
if (renamed && (back->status == STATUS_READY ||
|
||||
(back->r.out = FOPEN(newname, "ab")) != NULL)) {
|
||||
filenote(&opt->state.strc, newname, NULL);
|
||||
hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
|
||||
newname);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
/* partial lost: drop only what we own (Windows rename won't overwrite) */
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
|
||||
back->url_sav, newname);
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(back->r.msg, "Write error on disk");
|
||||
back->r.is_write = 0;
|
||||
(void) UNLINK(renamed ? newname : back->url_sav);
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
// effacer entrée
|
||||
/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
|
||||
static void back_delayed_discard(httrackp *opt, lien_back *back) {
|
||||
void back_delayed_discard(httrackp *opt, lien_back *back) {
|
||||
if (back->r.out != NULL) {
|
||||
fclose(back->r.out);
|
||||
back->r.out = NULL;
|
||||
@@ -1403,7 +1433,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
back[p].url_adr, back[p].url_fil, back[p].url_sav);
|
||||
}
|
||||
if (cache != NULL) {
|
||||
//hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
|
||||
back_finalize(opt, cache, sback, p);
|
||||
}
|
||||
}
|
||||
@@ -1544,7 +1573,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
if (back[p].r.soc != INVALID_SOCKET) { /* we never know */
|
||||
deletehttp(&back[p].r);
|
||||
}
|
||||
//memset(&(back[p].r), 0, sizeof(htsblk));
|
||||
hts_init_htsblk(&back[p].r);
|
||||
back[p].r.location = back[p].location_buffer;
|
||||
|
||||
@@ -1552,7 +1580,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
strcpybuff(back[p].url_adr, adr);
|
||||
strcpybuff(back[p].url_fil, fil);
|
||||
strcpybuff(back[p].url_sav, save);
|
||||
//back[p].links_index = links_index;
|
||||
// copier referer si besoin
|
||||
strcpybuff(back[p].referer_adr, "");
|
||||
strcpybuff(back[p].referer_fil, "");
|
||||
@@ -1791,8 +1818,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
back[p].url_adr, back[p].url_fil);
|
||||
}
|
||||
back[p].r.notmodified = 1; // fichier non modifié
|
||||
back[p].status = STATUS_READY; // OK prêt
|
||||
//file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); // not modified
|
||||
back[p].status = STATUS_READY; // OK prêt
|
||||
back_set_finished(sback, p);
|
||||
|
||||
// finalize transfer
|
||||
@@ -1807,7 +1833,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
} else { // erreur
|
||||
// effacer r
|
||||
hts_init_htsblk(&back[p].r);
|
||||
//memset(&(back[p].r), 0, sizeof(htsblk));
|
||||
back[p].r.location = back[p].location_buffer;
|
||||
// et continuer (chercher le fichier)
|
||||
}
|
||||
@@ -2002,7 +2027,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
// ouvrir liaison, envoyer requète
|
||||
// ne pas traiter ou recevoir l'en tête immédiatement
|
||||
hts_init_htsblk(&back[p].r);
|
||||
// memset(&(back[p].r), 0, sizeof(htsblk));
|
||||
back[p].r.location = back[p].location_buffer;
|
||||
// fresh connect: address list not yet probed, start at the first
|
||||
sback->connect_fallback[p].addr_index = 0;
|
||||
@@ -2053,7 +2077,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
#if HTS_USEOPENSSL
|
||||
else if (strfield(back[p].url_adr, "https://")) { // let's rock
|
||||
back[p].r.ssl = 1;
|
||||
// back[p].r.ssl_soc = NULL;
|
||||
back[p].r.ssl_con = NULL;
|
||||
}
|
||||
#endif
|
||||
@@ -2119,9 +2142,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
back[p].rateout = -1; // pas de gestion (default)
|
||||
}
|
||||
|
||||
// Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
|
||||
// rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
|
||||
//if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
|
||||
// Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
|
||||
// cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
|
||||
// page!)
|
||||
|
||||
#if CNXDEBUG
|
||||
printf("Xfopen ok, poll..\n");
|
||||
@@ -2139,7 +2162,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
if (soc == INVALID_SOCKET) { // erreur socket
|
||||
back[p].status = STATUS_READY; // FINI
|
||||
back_set_finished(sback, p);
|
||||
//if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
|
||||
back[p].r.soc = INVALID_SOCKET;
|
||||
} else {
|
||||
if (!back[p].r.is_file)
|
||||
@@ -2160,8 +2182,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
|
||||
// note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
|
||||
// le lien est considéré comme traité
|
||||
//if (back[p].soc<0) // erreur
|
||||
// return -1;
|
||||
|
||||
return 0;
|
||||
} else {
|
||||
@@ -2234,9 +2254,6 @@ void back_solve(httrackp * opt, lien_back * back) {
|
||||
} else {
|
||||
hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
|
||||
}
|
||||
//if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
|
||||
// hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
|
||||
//}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2285,11 +2302,6 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
|
||||
(void) back_flush_output(opt, cache, sback, i); // flush output buffers
|
||||
usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
//if (back[i].links_index >= 0) {
|
||||
// assertf(back[i].links_index < opt->hash->max_lien);
|
||||
// opt->hash->liens[back[i].links_index]->pass2 = -1;
|
||||
// // *back[i].pass2_ptr=-1; // Done!
|
||||
//}
|
||||
/* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
|
||||
{
|
||||
int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME ); // lecture type 0 (sav)
|
||||
@@ -2473,9 +2485,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
|
||||
busy_state = busy_recv = 0;
|
||||
|
||||
#if 0
|
||||
check_rate(stat_timestart, opt->maxrate); // vérifier taux de transfert
|
||||
#endif
|
||||
// inscrire les sockets actuelles, et rechercher l'ID la plus élevée
|
||||
FD_ZERO(&fds);
|
||||
FD_ZERO(&fds_c);
|
||||
@@ -2485,8 +2494,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
nfds = INVALID_SOCKET;
|
||||
|
||||
max_c = 1;
|
||||
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
// for(i=0;i<back_max;i++) {
|
||||
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
unsigned int i = (i_mod + mod_random) % (back_max);
|
||||
|
||||
// en cas de gestion du connect préemptif
|
||||
@@ -2523,8 +2531,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// poll pour la lecture sur les sockets
|
||||
if ((back[i].status > 0) && (back[i].status < 100)) { // en réception http
|
||||
|
||||
#if BDEBUG==1
|
||||
//printf("....socket in progress: %d\n",back[i].r.soc);
|
||||
#if BDEBUG == 1
|
||||
#endif
|
||||
// non local et non ftp
|
||||
if (!back[i].r.is_file) {
|
||||
@@ -2610,8 +2617,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
busy_recv = 0;
|
||||
|
||||
// recevoir les données arrivées
|
||||
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
// for(i=0;i<back_max;i++) {
|
||||
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
unsigned int i = (i_mod + mod_random) % (back_max);
|
||||
|
||||
if (back[i].status > 0) {
|
||||
@@ -2751,7 +2757,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].rateout_time = back[i].ka_time_start;
|
||||
}
|
||||
// envoyer header
|
||||
//if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0) // vrai get
|
||||
HTS_STAT.stat_nrequests++;
|
||||
if (!back[i].head_request)
|
||||
http_sendhead(opt, opt->cookie, 0, back[i].send_too,
|
||||
@@ -2818,7 +2823,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
#if HTS_XGETHOST
|
||||
else if (back[i].status == STATUS_WAIT_DNS) { // attendre gethostbyname
|
||||
#if DEBUGDNS
|
||||
//printf("status 101 for %s\n",back[i].url_adr);
|
||||
#endif
|
||||
|
||||
if (!gestion_timeout)
|
||||
@@ -3091,8 +3095,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// Si réception chunk, tester si on est pas à la fin!
|
||||
if (back[i].status == 1) {
|
||||
if (back[i].is_chunk) { // attendre prochain chunk
|
||||
if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
|
||||
//printf("chunk end at %d\n",back[i].r.size);
|
||||
if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
|
||||
back[i].status = STATUS_CHUNK_CR; /* fetch ending CRLF */
|
||||
if (back[i].chunk_adr != NULL) {
|
||||
freet(back[i].chunk_adr);
|
||||
@@ -3155,11 +3158,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back_finalize(opt, cache, sback, i);
|
||||
}
|
||||
|
||||
if (back[i].r.totalsize >= 0) { // tester totalsize
|
||||
//if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) { // tester totalsize
|
||||
if (back[i].r.totalsize >= 0) { // tester totalsize
|
||||
if (back[i].r.totalsize != back[i].r.size) { // pas la même!
|
||||
if (!opt->tolerant) {
|
||||
//#if HTS_CL_IS_FATAL
|
||||
deleteaddr(&back[i].r);
|
||||
if (back[i].r.size < back[i].r.totalsize)
|
||||
back[i].r.statuscode = STATUSCODE_CONNERROR; // recatch
|
||||
@@ -3168,14 +3169,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
" expected)", (LLint) back[i].r.size,
|
||||
(LLint) back[i].r.totalsize);
|
||||
} else {
|
||||
//#else
|
||||
// Un warning suffira..
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Incorrect length (" LLintP "!=" LLintP
|
||||
" expected) for %s%s", (LLint) back[i].r.size,
|
||||
" expected) for %s%s",
|
||||
(LLint) back[i].r.size,
|
||||
(LLint) back[i].r.totalsize, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
//#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3302,10 +3302,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].r.size);
|
||||
#endif
|
||||
/* End */
|
||||
//if (back[i].status==STATUS_CHUNK_CR) {
|
||||
back[i].status = STATUS_READY; // fin
|
||||
back[i].status = STATUS_READY; // fin
|
||||
back_set_finished(sback, i);
|
||||
//}
|
||||
|
||||
// finalize transfer if not temporary
|
||||
if (!IS_DELAYED_EXT(back[i].url_sav)) {
|
||||
@@ -3772,7 +3770,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
|
||||
// finalize
|
||||
//file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); // not modified
|
||||
if (back[i].r.statuscode > 0) {
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
@@ -3782,37 +3779,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
#endif
|
||||
|
||||
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
|
||||
} else { // erreur
|
||||
back[i].status = STATUS_READY; // terminé
|
||||
back_set_finished(sback, i);
|
||||
//printf("erreur cache\n");
|
||||
|
||||
}
|
||||
|
||||
/********** NO - must complete the body! ********** */
|
||||
#if 0
|
||||
} else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
|
||||
|| (back[i].r.statuscode == 412)
|
||||
|| (back[i].r.statuscode == 416)
|
||||
) { // Ne pas prendre le html, erreurs connues et gérées
|
||||
#if HTS_DEBUG_CLOSESOCK
|
||||
DEBUG_W
|
||||
("back_wait(301,302,303,307,412,416..): deletehttp\n");
|
||||
#endif
|
||||
// Couper connexion
|
||||
/*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
|
||||
back_maydeletehttp(opt, cache, sback, i);
|
||||
|
||||
back[i].status = STATUS_READY; // terminé
|
||||
back_set_finished(sback, i);
|
||||
// finalize
|
||||
if (back[i].r.statuscode > 0) {
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
}
|
||||
#endif
|
||||
/********** **************************** ********** */
|
||||
}
|
||||
// MIME type excluded by a -mime: filter: abort, don't fetch
|
||||
// the body (#58)
|
||||
@@ -3986,7 +3957,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
deletehttp(&back[i].r);
|
||||
}
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
//back[i].r.statuscode=206; ????????
|
||||
back[i].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
if (strnotempty(back[i].r.msg))
|
||||
strcpybuff(back[i].r.msg,
|
||||
@@ -4013,8 +3983,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing empty");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
}
|
||||
} else if (!back[i].r.is_chunk) { // pas de chunk
|
||||
//if (back[i].r.http11!=2) { // pas de chunk
|
||||
} else if (!back[i].r.is_chunk) { // pas de chunk
|
||||
back[i].is_chunk = 0;
|
||||
back[i].status = 1; // start body
|
||||
} else {
|
||||
@@ -4075,14 +4044,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (back[i].status < 0) {
|
||||
if (!back[i].testmode) { // pas en test
|
||||
UNLINK(back[i].url_sav); // éliminer fichier (endommagé)
|
||||
//printf("&& %s\n",back[i].url_sav);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* funny log for commandline users */
|
||||
//if (!opt->quiet) {
|
||||
// petite animation
|
||||
if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
|
||||
if (back[i].status == STATUS_READY) {
|
||||
if (back[i].r.statuscode == HTTP_OK)
|
||||
@@ -4095,18 +4061,16 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
//}
|
||||
|
||||
} // status>0
|
||||
} // for
|
||||
} // for
|
||||
|
||||
// vérifier timeouts
|
||||
if (gestion_timeout) {
|
||||
TStamp act;
|
||||
|
||||
act = time_local(); // temps en secondes
|
||||
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
// for(i=0;i<back_max;i++) {
|
||||
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
unsigned int i = (i_mod + mod_random) % (back_max);
|
||||
|
||||
if (back[i].status > 0) { // réception/connexion/..
|
||||
@@ -4134,7 +4098,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
continue;
|
||||
}
|
||||
}
|
||||
//printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
|
||||
if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
|
||||
hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
|
||||
@@ -113,6 +113,12 @@ void back_set_locked(struct_back * sback, const int p);
|
||||
void back_set_unlocked(struct_back * sback, const int p);
|
||||
int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p);
|
||||
/* Discard back's on-disk .delayed placeholder and its refname. */
|
||||
void back_delayed_discard(httrackp *opt, lien_back *back);
|
||||
/* Move back's .delayed placeholder (and open stream) to newname;
|
||||
HTS_FALSE = file lost, slot flagged in error. */
|
||||
hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
|
||||
const char *newname);
|
||||
void back_index_unlock(struct_back * sback, const int p);
|
||||
int back_clear_entry(lien_back * back);
|
||||
int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
|
||||
@@ -55,8 +55,6 @@ Please visit our Website: http://www.httrack.com
|
||||
// inline function in Wspiapi.h.
|
||||
#include <ws2tcpip.h>
|
||||
#include <Wspiapi.h>
|
||||
// #include <winsock2.h>
|
||||
// #include <tpipv6.h>
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
|
||||
#if DEBUG_COOK
|
||||
printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
|
||||
cook_name, cook_value, domain, path);
|
||||
//printf(">>>cook: %s<<<\n",cookie->data);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
char buffer[8192];
|
||||
|
||||
// cookie->data[0]='\0';
|
||||
|
||||
// Fusionner d'abord les éventuels cookies IE
|
||||
#ifdef _WIN32
|
||||
{
|
||||
|
||||
249
src/htscache.c
249
src/htscache.c
@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htsbasenet.h"
|
||||
#include "htsmd5.h"
|
||||
#include <limits.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "htszlib.h"
|
||||
@@ -220,23 +221,38 @@ struct cache_back_zip_entry {
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* A cache (new.zip) write failed: storage is gone (disk full / dropped share),
|
||||
so the mirror is doomed too. Abort it via exit_xh, don't crash as assertf
|
||||
did. */
|
||||
/* Consecutive entry write failures before the cache stream is declared dead. */
|
||||
#define CACHE_MAX_WRITE_FAILURES 8
|
||||
|
||||
/* Cache write failed: a fatal errno or a failure streak aborts the mirror
|
||||
(exit_xh); an isolated failure only drops the current entry. */
|
||||
static void cache_zip_write_failed(httrackp *opt, cache_back *cache,
|
||||
const char *what, int zErr) {
|
||||
if (!cache->zipWriteFailed) {
|
||||
cache->zipWriteFailed = HTS_TRUE;
|
||||
if (check_fatal_io_errno()) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: disk full or filesystem problems");
|
||||
} else {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: cache write failed (%s): %s", what,
|
||||
hts_get_zerror(zErr));
|
||||
const char *what, int zErr,
|
||||
hts_boolean entry_open, const char *url_adr,
|
||||
const char *url_fil) {
|
||||
const int fatal_errno = zErr == ZIP_ERRNO && check_fatal_io_errno();
|
||||
|
||||
cache->zipWriteFailures++;
|
||||
if (fatal_errno || cache->zipWriteFailures >= CACHE_MAX_WRITE_FAILURES) {
|
||||
if (!cache->zipWriteFailed) {
|
||||
cache->zipWriteFailed = HTS_TRUE;
|
||||
if (fatal_errno) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: disk full or filesystem problems");
|
||||
} else {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: cache write failed (%s): %s", what,
|
||||
hts_get_zerror(zErr));
|
||||
}
|
||||
}
|
||||
opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
|
||||
} else {
|
||||
if (entry_open)
|
||||
zipCloseFileInZip((zipFile) cache->zipOutput); /* abandon, best-effort */
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"cache write failed (%s: %s), entry not cached: %s%s", what,
|
||||
hts_get_zerror(zErr), url_adr, url_fil);
|
||||
}
|
||||
opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
|
||||
}
|
||||
|
||||
/* Ajout d'un fichier en cache */
|
||||
@@ -249,8 +265,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
char BIGSTK headers[8192];
|
||||
int headersSize = 0;
|
||||
|
||||
//int entryBodySize = 0;
|
||||
//int entryFilenameSize = 0;
|
||||
zip_fileinfo fi;
|
||||
const char *url_save_suffix = url_save;
|
||||
int zErr;
|
||||
@@ -286,10 +300,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if (r->size < 0) // error
|
||||
return;
|
||||
|
||||
// data in cache
|
||||
if (dataincache) {
|
||||
assertf(((int) r->size) == r->size);
|
||||
//entryBodySize = (int) r->size;
|
||||
// data in cache: the body must fit the 32-bit zip write API
|
||||
if (dataincache && (LLint) (int) r->size != r->size) {
|
||||
if (r->is_write && url_save != NULL && strnotempty(url_save)) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"file too large for the cache, storing headers only: %s%s",
|
||||
url_adr, url_fil);
|
||||
dataincache = 0;
|
||||
} else {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"entry too large for the cache, not cached: %s%s", url_adr,
|
||||
url_fil);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/* Fields */
|
||||
@@ -334,8 +357,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename
|
||||
ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix); // Original save filename
|
||||
|
||||
//entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil));
|
||||
|
||||
/* Filename */
|
||||
if (!link_has_authority(url_adr)) {
|
||||
strcpybuff(filename, "http://");
|
||||
@@ -369,7 +390,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
*/
|
||||
headers, (uInt) strlen(headers), NULL, 0, NULL, /* comment */
|
||||
Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "opening a cache entry", zErr);
|
||||
cache_zip_write_failed(opt, cache, "opening a cache entry", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -380,7 +402,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if ((zErr =
|
||||
zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr,
|
||||
(int) r->size)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr);
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
|
||||
HTS_TRUE, url_adr, url_fil);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -402,8 +425,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if ((zErr =
|
||||
zipWriteInFileInZip((zipFile) cache->zipOutput, buff,
|
||||
(int) nl)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache",
|
||||
zErr);
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
|
||||
HTS_TRUE, url_adr, url_fil);
|
||||
fclose(fp);
|
||||
return;
|
||||
}
|
||||
@@ -419,15 +442,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
|
||||
/* Close */
|
||||
if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "closing a cache entry", zErr);
|
||||
cache_zip_write_failed(opt, cache, "closing a cache entry", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Flush */
|
||||
if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) {
|
||||
cache_zip_write_failed(opt, cache, "flushing the cache", zErr);
|
||||
cache_zip_write_failed(opt, cache, "flushing the cache", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
return;
|
||||
}
|
||||
|
||||
cache->zipWriteFailures = 0; /* entry stored: reset the failure streak */
|
||||
}
|
||||
|
||||
#else
|
||||
@@ -694,7 +721,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
char BIGSTK headerBuff[8192 + 2];
|
||||
int readSizeHeader;
|
||||
|
||||
//int totalHeader = 0;
|
||||
int dataincache = 0;
|
||||
|
||||
/* For BIG comments */
|
||||
@@ -745,13 +771,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
HTS_URLMAXSIZE * 2);
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo,
|
||||
sizeof(r.cdispo));
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
|
||||
sizeof(previous_save_));
|
||||
}
|
||||
} while(offset < readSizeHeader && !lineEof);
|
||||
//totalHeader = offset;
|
||||
} while (offset < readSizeHeader && !lineEof);
|
||||
|
||||
/* Previous entry */
|
||||
if (previous_save_[0] != '\0') {
|
||||
@@ -768,6 +791,15 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
|
||||
}
|
||||
|
||||
/* A negative X-Size is corrupt; so is one >= INT_MAX when the data
|
||||
is in the zip (the write path asserts int-sized). Headers-only
|
||||
entries legitimately exceed INT_MAX (>2GB body on disk): keep
|
||||
them, or every update would re-fetch the file. */
|
||||
if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Read Error : Bad Size");
|
||||
}
|
||||
|
||||
/* Complete fields */
|
||||
r.totalsize = r.size;
|
||||
r.adr = NULL;
|
||||
@@ -794,7 +826,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
} // otherwise, the ZIP file is supposed to be consistent with data.
|
||||
}
|
||||
/* Read data ? */
|
||||
else { /* ne pas lire uniquement header */
|
||||
else if (r.statuscode !=
|
||||
STATUSCODE_INVALID) { /* ne pas lire uniquement header */
|
||||
int ok = 0;
|
||||
|
||||
#if HTS_DIRECTDISK
|
||||
@@ -923,7 +956,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg,
|
||||
"Cache Write Error : Unable to Create File");
|
||||
//printf("%s\n",save);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -958,7 +990,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strcpybuff(r.msg,
|
||||
"Previous cache file not found (empty filename)");
|
||||
}
|
||||
} else { /* Read in memory from disk */
|
||||
} else if (r.size >= INT_MAX) { /* too big to read in memory */
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Read Error : Bad Size");
|
||||
} else { /* Read in memory from disk */
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
|
||||
|
||||
if (fp != NULL) {
|
||||
@@ -998,7 +1033,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strcpybuff(r.msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
*(r.adr + r.size) = '\0';
|
||||
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
|
||||
} else { // erreur
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Memory Error");
|
||||
@@ -1177,17 +1211,14 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
// sécurité
|
||||
r.adr = NULL;
|
||||
r.out = NULL;
|
||||
////r.location=NULL; non, fixée lors des 301 ou 302
|
||||
r.fp = NULL;
|
||||
|
||||
if ((r.statuscode >= 0) && (r.statuscode <= 999)
|
||||
&& (r.notmodified >= 0) && (r.notmodified <= 9)) { // petite vérif intégrité
|
||||
if ((save) && (!header_only)) { /* ne pas lire uniquement header */
|
||||
//int to_file=0;
|
||||
if ((save) && (!header_only)) { /* ne pas lire uniquement header */
|
||||
|
||||
r.adr = NULL;
|
||||
r.soc = INVALID_SOCKET;
|
||||
// // r.location=NULL;
|
||||
|
||||
#if HTS_DIRECTDISK
|
||||
// Court-circuit:
|
||||
@@ -1196,12 +1227,11 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
int ok = 0;
|
||||
|
||||
r.is_write = 1; // écrire
|
||||
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) { // un fichier existe déja
|
||||
//if (fsize_utf8(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree)
|
||||
if (fexist_utf8(fconv(catbuff, sizeof(catbuff),
|
||||
save))) { // un fichier existe déja
|
||||
ok = 1; // plus rien à faire
|
||||
filenote(&opt->state.strc, save, NULL); // noter comme connu
|
||||
file_notify(opt, adr, fil, save, 0, 0, 0);
|
||||
//}
|
||||
}
|
||||
|
||||
if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur!
|
||||
@@ -1252,7 +1282,6 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg,
|
||||
"Cache Write Error : Unable to Create File");
|
||||
//printf("%s\n",save);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1300,14 +1329,13 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
strcpybuff(r.msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
*(r.adr + r.size) = '\0';
|
||||
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
|
||||
} else { // erreur
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Memory Error");
|
||||
}
|
||||
}
|
||||
}
|
||||
} // si save==null, ne rien charger (juste en tête)
|
||||
} // si save==null, ne rien charger (juste en tête)
|
||||
} else {
|
||||
#if DEBUGCA
|
||||
printf("Cache Read Error : Bad Data");
|
||||
@@ -1420,6 +1448,86 @@ static int hts_rename(httrackp * opt, const char *a, const char *b) {
|
||||
return rename(a, b);
|
||||
}
|
||||
|
||||
/* Pathname of a file inside the mirror dir (rotating concat buffer). */
|
||||
static char *reconcile_path(httrackp *opt, const char *name) {
|
||||
return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), name);
|
||||
}
|
||||
|
||||
/* Interrupted-run heuristic: prefer the old generation when the new cache
|
||||
stalled below NEW_TINY while the old one grew past OLD_SOLID (historical
|
||||
arbitrary thresholds). */
|
||||
#define CACHE_RECONCILE_NEW_TINY 32768
|
||||
#define CACHE_RECONCILE_OLD_SOLID 65536
|
||||
|
||||
/* Replace the new-generation file by the old one, when the old one exists. */
|
||||
static void reconcile_promote(httrackp *opt, const char *oldname,
|
||||
const char *newname) {
|
||||
if (fexist(reconcile_path(opt, oldname))) {
|
||||
remove(reconcile_path(opt, newname));
|
||||
rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
|
||||
}
|
||||
}
|
||||
|
||||
void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
|
||||
switch (mode) {
|
||||
case CACHE_RECONCILE_PROMOTE:
|
||||
/* Previous run rotated new.* to old.* then died before writing: promote
|
||||
the old generation back, whichever format it uses. */
|
||||
if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if ((!fexist(reconcile_path(opt, "hts-cache/new.dat")) ||
|
||||
!fexist(reconcile_path(opt, "hts-cache/new.ndx"))) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
break;
|
||||
case CACHE_RECONCILE_INTERRUPTED:
|
||||
/* Aborted run: keep the larger generation when the new cache is
|
||||
suspiciously small next to the old one. The new file must exist: fsize()
|
||||
is -1 for a missing file, which would spuriously pass the "< TINY" test
|
||||
and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
|
||||
*/
|
||||
if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
|
||||
break;
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.dat")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.dat")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.dat")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.dat"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
break;
|
||||
case CACHE_RECONCILE_ROLLBACK:
|
||||
/* Nothing transferred: restore the previous generation and sidecars. */
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if (fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
reconcile_promote(opt, "hts-cache/old.lst", "hts-cache/new.lst");
|
||||
reconcile_promote(opt, "hts-cache/old.txt", "hts-cache/new.txt");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// renvoyer uniquement en tête, ou NULL si erreur
|
||||
// return NULL upon error, and set -1 to r.statuscode
|
||||
htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
|
||||
@@ -1450,25 +1558,10 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
|
||||
HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) { // il existe déja un cache précédent.. renommer
|
||||
/* Previous cache from the previous cache version */
|
||||
#if 0
|
||||
/* No.. reuse with old httrack releases! */
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"));
|
||||
#endif
|
||||
if ((fexist(fconcat(
|
||||
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))) { // a previous cache exists.. rename it
|
||||
/* Previous cache version */
|
||||
if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
|
||||
rename(fconcat
|
||||
@@ -1509,7 +1602,13 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
} else {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
|
||||
}
|
||||
} else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
|
||||
} else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))) &&
|
||||
(fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))) { // a previous cache
|
||||
// exists.. rename it
|
||||
#if DEBUGCA
|
||||
printf("work with former cache\n");
|
||||
#endif
|
||||
@@ -1538,7 +1637,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"));
|
||||
} else { // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel
|
||||
} else { // one or both cache files missing: remove the remaining one
|
||||
#if DEBUGCA
|
||||
printf("new cache\n");
|
||||
#endif
|
||||
@@ -1907,12 +2006,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"hts-cache/new.lst"), "wb");
|
||||
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
|
||||
opt->state.strc.lst = cache->lst;
|
||||
//{
|
||||
//filecreate_params tmp;
|
||||
//strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin
|
||||
//tmp.lst=cache->lst; // fichier lst
|
||||
//filenote("",&tmp); // initialiser filecreate
|
||||
//}
|
||||
|
||||
// supprimer old.txt
|
||||
if (fexist
|
||||
@@ -2001,12 +2094,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"hts-cache/new.lst"), "wb");
|
||||
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
|
||||
opt->state.strc.lst = cache->lst;
|
||||
//{
|
||||
// filecreate_params tmp;
|
||||
// strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin
|
||||
// tmp.lst=cache->lst; // fichier lst
|
||||
// filenote("",&tmp); // initialiser filecreate
|
||||
//}
|
||||
|
||||
// supprimer old.txt
|
||||
if (fexist
|
||||
@@ -2038,8 +2125,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"
|
||||
LF);
|
||||
}
|
||||
// test
|
||||
// cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9);
|
||||
} // cache->ndx!=NULL
|
||||
} //cache->zipOutput != NULL
|
||||
|
||||
|
||||
@@ -78,6 +78,17 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, htsblk * r);
|
||||
void cache_init(cache_back * cache, httrackp * opt);
|
||||
|
||||
/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
|
||||
typedef enum {
|
||||
CACHE_RECONCILE_PROMOTE, /* no new cache: promote the old generation */
|
||||
CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
|
||||
CACHE_RECONCILE_ROLLBACK /* nothing transferred: restore the old one */
|
||||
} hts_cache_reconcile_mode;
|
||||
|
||||
/* Reconcile the on-disk cache generations according to mode; a no-op when
|
||||
the involved files are absent. */
|
||||
void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
|
||||
|
||||
int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
|
||||
const char *str2, char *outbuff, int len);
|
||||
int cache_readdata(cache_back * cache, const char *str1, const char *str2,
|
||||
|
||||
@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htszlib.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
@@ -321,6 +322,7 @@ typedef struct {
|
||||
size_t budget; /**< bytes allowed through before writes start failing */
|
||||
int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
|
||||
int writes; /**< zwrite call count, to detect re-entry into the stream */
|
||||
int fail_once; /**< recover (unlimited budget) after the first failure */
|
||||
} writefail_inject;
|
||||
|
||||
/* zwrite that copies until the budget runs out, then fails with inj->fail_errno
|
||||
@@ -335,6 +337,8 @@ static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
|
||||
inj->budget -= (size_t) size;
|
||||
return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
|
||||
}
|
||||
if (inj->fail_once)
|
||||
inj->budget = (size_t) -1; /* the backend recovers after this failure */
|
||||
errno = inj->fail_errno;
|
||||
return 0; /* short write -> the minizip op returns an error */
|
||||
}
|
||||
@@ -373,9 +377,50 @@ static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
|
||||
freet(bodycopy);
|
||||
}
|
||||
|
||||
/* #174/#219: a failing cache write used to crash via assertf(); it must instead
|
||||
stop the mirror (exit_xh = -1) without crashing. Assert that, plus the cache
|
||||
is flagged and a sibling write doesn't re-enter the broken stream. */
|
||||
/* Store an entry claiming a >2GB body; the degrade path never reads data. */
|
||||
static void writefail_store_oversized(httrackp *opt, cache_back *cache,
|
||||
const char *fil, int is_write) {
|
||||
htsblk r;
|
||||
char locbuf[4];
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
r.statuscode = 200;
|
||||
r.size = (LLint) INT_MAX + 1;
|
||||
strcpybuff(r.msg, "OK");
|
||||
strcpybuff(r.contenttype, "application/octet-stream");
|
||||
locbuf[0] = '\0';
|
||||
r.location = locbuf;
|
||||
r.is_write = (short int) is_write;
|
||||
cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
|
||||
}
|
||||
|
||||
/* Read back `entryname`: extra field (cached headers) and body. Returns the
|
||||
body length, or -1 if the entry is absent or unreadable. */
|
||||
static int writefail_read_entry(const char *path, const char *entryname,
|
||||
char *extra, size_t extralen, char *body,
|
||||
size_t bodylen) {
|
||||
unzFile z = unzOpen(path);
|
||||
int n = -1;
|
||||
|
||||
if (z == NULL)
|
||||
return -1;
|
||||
if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
|
||||
unzOpenCurrentFile(z) == UNZ_OK) {
|
||||
const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
|
||||
|
||||
if (elen >= 0) {
|
||||
extra[elen] = '\0';
|
||||
n = unzReadCurrentFile(z, body, (unsigned) bodylen);
|
||||
}
|
||||
unzCloseCurrentFile(z);
|
||||
}
|
||||
unzClose(z);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
|
||||
stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
|
||||
*/
|
||||
int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
int fail = 0;
|
||||
char path[HTS_URLMAXSIZE];
|
||||
@@ -388,9 +433,8 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
gen_body(body, body_len, 1 /* incompressible */);
|
||||
fconcat(path, sizeof(path), dir, "/wfail.zip");
|
||||
|
||||
/* phase 0: fail on the body write, fatal errno (ENOSPC, the disk-full
|
||||
branch). phase 1: fail on the open, non-fatal errno (EIO, dropped-share
|
||||
branch). Both must abort the mirror. */
|
||||
/* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
|
||||
drops entries until the streak caps out, then aborts. */
|
||||
for (phase = 0; phase < 2; phase++) {
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
@@ -399,6 +443,7 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
inj.budget = (phase == 0) ? 4096 : 0;
|
||||
inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
@@ -412,7 +457,25 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
}
|
||||
|
||||
opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
if (phase == 0) {
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
} else {
|
||||
/* the abort must land exactly on the 8th consecutive failure */
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 7; i++) {
|
||||
char fil[32];
|
||||
|
||||
snprintf(fil, sizeof(fil), "/b%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
}
|
||||
if (cache.zipWriteFailed) {
|
||||
fprintf(stderr, "cache-writefail: phase 1: aborted before the "
|
||||
"8th consecutive failure\n");
|
||||
fail++;
|
||||
}
|
||||
writefail_store(opt, &cache, "/b7.bin", body, 16);
|
||||
}
|
||||
if (!cache.zipWriteFailed) {
|
||||
fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
|
||||
phase);
|
||||
@@ -443,6 +506,136 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
}
|
||||
}
|
||||
|
||||
/* failures with successes in between reset the streak: never aborts */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
int i;
|
||||
|
||||
inj.budget = (size_t) -1;
|
||||
inj.fail_errno = EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
for (i = 0; i < 10; i++) {
|
||||
char fil[32];
|
||||
|
||||
inj.budget = 0; /* this store fails */
|
||||
snprintf(fil, sizeof(fil), "/s%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
|
||||
snprintf(fil, sizeof(fil), "/ok%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
}
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: scattered: non-consecutive failures aborted "
|
||||
"the mirror (flagged=%d, exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
}
|
||||
|
||||
/* isolated failure: only that entry drops; a later sibling round-trips */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
char extra[8192];
|
||||
char rbody[64];
|
||||
int n;
|
||||
|
||||
inj.budget = 4096;
|
||||
inj.fail_errno = EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 1;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: skip: isolated failure aborted the mirror "
|
||||
"(flagged=%d, exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
writefail_store(opt, &cache, "/blob2.bin", body, 16);
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody));
|
||||
if (n != 16 || memcmp(rbody, body, 16) != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: skip: sibling entry lost after a skipped "
|
||||
"entry (%d)\n",
|
||||
n);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
|
||||
/* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
char extra[8192];
|
||||
char rbody[64];
|
||||
int n;
|
||||
|
||||
inj.budget = (size_t) -1; /* no injected failure */
|
||||
inj.fail_errno = 0;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
|
||||
writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: mirror aborted (flagged=%d, "
|
||||
"exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody)) >= 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: in-memory entry was stored\n");
|
||||
fail++;
|
||||
}
|
||||
n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody));
|
||||
if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: on-disk entry not stored "
|
||||
"headers-only (%d)\n",
|
||||
n);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
|
||||
freet(body);
|
||||
return fail;
|
||||
}
|
||||
@@ -716,3 +909,494 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* --- hts_cache_reconcile() policies -------------------------------------- */
|
||||
|
||||
/* All reconcile inputs/outputs, wiped between cases. */
|
||||
static const char *const reconcile_files[] = {
|
||||
"hts-cache/new.zip", "hts-cache/old.zip", "hts-cache/new.dat",
|
||||
"hts-cache/old.dat", "hts-cache/new.ndx", "hts-cache/old.ndx",
|
||||
"hts-cache/new.lst", "hts-cache/old.lst", "hts-cache/new.txt",
|
||||
"hts-cache/old.txt", "hts-in_progress.lock"};
|
||||
|
||||
static char *reconcile_st_path(httrackp *opt, const char *name) {
|
||||
return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), name);
|
||||
}
|
||||
|
||||
static void reconcile_wipe(httrackp *opt) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
|
||||
remove(reconcile_st_path(opt, reconcile_files[i]));
|
||||
}
|
||||
|
||||
/* Create a filler file of exactly `size` bytes. */
|
||||
static void reconcile_put(httrackp *opt, const char *name, size_t size) {
|
||||
FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
|
||||
static const char filler[1024] = {'x'};
|
||||
|
||||
assertf(fp != NULL);
|
||||
while (size > 0) {
|
||||
const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
|
||||
|
||||
assertf(fwrite(filler, 1, n, fp) == n);
|
||||
size -= n;
|
||||
}
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
|
||||
static int reconcile_expect(httrackp *opt, const char *name, off_t size,
|
||||
const char *what) {
|
||||
const off_t got = fsize(reconcile_st_path(opt, name));
|
||||
|
||||
if (got != size) {
|
||||
fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
|
||||
name, (int) got, (int) size);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cache_reconcile_selftest(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
|
||||
/* around the interrupted-run thresholds (new < 32768, old > 65536) */
|
||||
static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
|
||||
|
||||
golden_setup(opt, dir);
|
||||
#ifdef _WIN32
|
||||
mkdir(reconcile_st_path(opt, "hts-cache"));
|
||||
#else
|
||||
mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
|
||||
/* PROMOTE: a zip old generation replaces a missing new one */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
|
||||
|
||||
/* PROMOTE: an existing new.zip is left alone */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
|
||||
|
||||
/* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
|
||||
zip cache existed) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
|
||||
|
||||
/* PROMOTE: a half-written legacy new pair is replaced by the old pair */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
|
||||
|
||||
/* INTERRUPTED: no lock file, no action */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
|
||||
|
||||
/* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
|
||||
spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
|
||||
|
||||
/* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
|
||||
zip caches: the arm was gated on a legacy new.dat) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
|
||||
|
||||
/* INTERRUPTED: old below the confidence threshold, keep new */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
|
||||
|
||||
/* INTERRUPTED: new big enough to trust, keep it */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", MID);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
|
||||
|
||||
/* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/new.ndx", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
|
||||
|
||||
/* ROLLBACK: the old zip generation is restored (a zip cache used to lose
|
||||
its only good generation here) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
|
||||
|
||||
/* ROLLBACK: sidecars are restored regardless of format */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.lst", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.lst", MID);
|
||||
reconcile_put(opt, "hts-cache/old.txt", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
|
||||
|
||||
/* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/new.ndx", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", MID);
|
||||
reconcile_put(opt, "hts-cache/old.lst", MID);
|
||||
reconcile_put(opt, "hts-cache/old.txt", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
|
||||
|
||||
/* ROLLBACK: nothing to restore, the new generation stays */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
|
||||
|
||||
reconcile_wipe(opt);
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* --- read-side corruption injection --------------------------------------- */
|
||||
|
||||
/* canary read back intact after each corruption; victim gets the byte surgery
|
||||
*/
|
||||
#define CORRUPT_ADR "corrupt.example.com"
|
||||
static char corrupt_body_a[33 + 1];
|
||||
static char corrupt_body_b[44 + 1];
|
||||
|
||||
/* Write a fresh two-entry cache: /canary.html then /victim.html. */
|
||||
static void corrupt_build(httrackp *opt) {
|
||||
cache_back cache;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
|
||||
strlen(corrupt_body_b));
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
|
||||
is later overwritten with a forged oversized X-Size (same byte length). */
|
||||
static void corrupt_build_etag(httrackp *opt) {
|
||||
cache_back cache;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
|
||||
corrupt_body_b, strlen(corrupt_body_b));
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
|
||||
body on disk): the shape every non-html file is stored with. */
|
||||
static void corrupt_build_disk(httrackp *opt) {
|
||||
cache_back cache;
|
||||
htsblk w;
|
||||
char locw[4];
|
||||
char BIGSTK save[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
|
||||
char *path;
|
||||
FILE *fp;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
|
||||
CORRUPT_ADR "/victim.bin");
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
hts_init_htsblk(&w);
|
||||
w.statuscode = 200;
|
||||
w.size = (LLint) sizeof(corrupt_body_b) - 1;
|
||||
strcpybuff(w.msg, "OK");
|
||||
strcpybuff(w.contenttype, "application/octet-stream");
|
||||
strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
|
||||
locw[0] = '\0';
|
||||
w.location = locw;
|
||||
w.is_write = 0;
|
||||
cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
|
||||
0 /* all_in_cache */, NULL);
|
||||
selftest_close(&cache);
|
||||
/* the reader only checks this file exists; it never reads it here */
|
||||
path = fconv(catbuff, sizeof(catbuff), save);
|
||||
(void) structcheck(path);
|
||||
fp = FOPEN(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
|
||||
static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
|
||||
const char *rep, size_t nth, size_t total) {
|
||||
LLint fsz = 0;
|
||||
char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
|
||||
const size_t n = (size_t) fsz;
|
||||
size_t k, hits = 0, at = 0;
|
||||
FILE *fp;
|
||||
|
||||
assertf(data != NULL);
|
||||
for (k = 0; k + patlen <= n; k++) {
|
||||
if (memcmp(data + k, pat, patlen) == 0) {
|
||||
hits++;
|
||||
if (hits == nth)
|
||||
at = k;
|
||||
}
|
||||
}
|
||||
assertf(hits == total);
|
||||
memcpy(data + at, rep, patlen);
|
||||
fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, n, fp) == n);
|
||||
fclose(fp);
|
||||
freet(data);
|
||||
}
|
||||
|
||||
/* Garbage the first bytes of the victim's deflated data (2nd local header). */
|
||||
static void corrupt_victim_body(httrackp *opt) {
|
||||
LLint fsz = 0;
|
||||
char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
|
||||
const size_t n = (size_t) fsz;
|
||||
size_t k, hits = 0, off = 0;
|
||||
FILE *fp;
|
||||
|
||||
assertf(data != NULL);
|
||||
for (k = 0; k + 4 <= n; k++) {
|
||||
if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
|
||||
const size_t namelen =
|
||||
(unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
|
||||
const size_t extralen =
|
||||
(unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
|
||||
|
||||
off = k + 30 + namelen + extralen;
|
||||
}
|
||||
}
|
||||
assertf(hits == 2);
|
||||
assertf(off != 0 && off + 4 <= n);
|
||||
memset(data + off, 0xFF, 4);
|
||||
fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, n, fp) == n);
|
||||
fclose(fp);
|
||||
freet(data);
|
||||
}
|
||||
|
||||
/* Read the corrupt /victim.html and, in the SAME read session, the intact
|
||||
/canary.html: the victim must be rejected (wantmsg pins which path) and the
|
||||
canary must still decode byte-exact, proving one bad entry never taints a
|
||||
sibling read. */
|
||||
static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
|
||||
const char *wantmsg, const char *what) {
|
||||
cache_back cache;
|
||||
htsblk v, c;
|
||||
char BIGSTK lv[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK lc[HTS_URLMAXSIZE * 2];
|
||||
int fail = 0;
|
||||
|
||||
selftest_open_for_read(&cache, opt);
|
||||
lv[0] = lc[0] = '\0';
|
||||
v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
|
||||
if (v.statuscode != STATUSCODE_INVALID) {
|
||||
fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
|
||||
selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
|
||||
fail++;
|
||||
}
|
||||
if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
|
||||
fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
|
||||
selftest_tag, what, v.msg, wantmsg);
|
||||
fail++;
|
||||
}
|
||||
c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
|
||||
if (c.statuscode != 200 || c.adr == NULL ||
|
||||
c.size != (LLint) strlen(corrupt_body_a) ||
|
||||
memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
|
||||
fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
|
||||
c.statuscode);
|
||||
fail++;
|
||||
}
|
||||
if (v.adr != NULL)
|
||||
freet(v.adr);
|
||||
if (c.adr != NULL)
|
||||
freet(c.adr);
|
||||
selftest_close(&cache);
|
||||
return fail;
|
||||
}
|
||||
|
||||
static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
|
||||
const char *what) {
|
||||
return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
|
||||
}
|
||||
|
||||
/* Headers-only probe of the disk victim: must parse OK with the size kept. */
|
||||
static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
|
||||
const char *what) {
|
||||
cache_back cache;
|
||||
htsblk v;
|
||||
char BIGSTK lv[HTS_URLMAXSIZE * 2];
|
||||
int fail = 0;
|
||||
|
||||
selftest_open_for_read(&cache, opt);
|
||||
lv[0] = '\0';
|
||||
v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
|
||||
if (v.statuscode != 200 || v.size != wantsize) {
|
||||
fprintf(stderr,
|
||||
"%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
|
||||
selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
|
||||
fail++;
|
||||
}
|
||||
if (v.adr != NULL)
|
||||
freet(v.adr);
|
||||
selftest_close(&cache);
|
||||
return fail;
|
||||
}
|
||||
|
||||
/* One zip corruption case: build, patch, then check victim+canary in-session.
|
||||
*/
|
||||
static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
|
||||
size_t nth, size_t total, const char *wantmsg,
|
||||
const char *what) {
|
||||
corrupt_build(opt);
|
||||
corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
|
||||
return corrupt_expect_victim(opt, wantmsg, what);
|
||||
}
|
||||
|
||||
int cache_corruption_selftest(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
|
||||
selftest_tag = "cache-corrupt";
|
||||
golden_setup(opt, dir);
|
||||
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
|
||||
"Cache Read Error : Read Data", "oversized X-Size");
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
|
||||
"Cache Read Error : Bad Size", "negative X-Size");
|
||||
/* both entries carry the line; the victim's is the second */
|
||||
failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
|
||||
"Previous cache file not found (empty filename)",
|
||||
"blanked X-In-Cache");
|
||||
/* smashed local file header: the entry is dropped at index load */
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
|
||||
"File Cache Entry Not Found", "smashed local header");
|
||||
|
||||
corrupt_build(opt);
|
||||
corrupt_victim_body(opt);
|
||||
failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
|
||||
"garbled deflate stream");
|
||||
|
||||
/* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
|
||||
truncates negative in the (int) cast the malloc uses: a wraparound alloc.
|
||||
cache_add asserts size fits an int, so such a value only reaches the reader
|
||||
from a corrupt/foreign cache; inject it by overwriting the victim's long
|
||||
Etag line with a same-length forged X-Size line (the parser keeps the last
|
||||
X-Size it sees), keeping the zip byte-length and offsets intact. */
|
||||
corrupt_build_etag(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483648AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
|
||||
"X-Size above INT_MAX");
|
||||
|
||||
/* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
|
||||
is how every >2GB non-html file is stored. It must survive a header probe
|
||||
(or every update re-fetches the file); an in-memory read still rejects. */
|
||||
corrupt_build_disk(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483648AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
|
||||
"headers-only X-Size above INT_MAX");
|
||||
failures += corrupt_expect_victim_fil(opt, "/victim.bin",
|
||||
"Cache Read Error : Bad Size",
|
||||
"in-memory X-Size above INT_MAX");
|
||||
|
||||
/* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
|
||||
corrupt_build_disk(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483647AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_victim_fil(opt, "/victim.bin",
|
||||
"Cache Read Error : Bad Size",
|
||||
"in-memory X-Size at INT_MAX");
|
||||
|
||||
/* the negative check must stay global, headers-only included */
|
||||
corrupt_build_disk(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: -2147483648AAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_victim_fil(opt, "/victim.bin",
|
||||
"Cache Read Error : Bad Size",
|
||||
"headers-only negative X-Size");
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
@@ -52,10 +52,19 @@ int cache_selftests(httrackp *opt, const char *dir);
|
||||
committed file, never by the test). Returns the failed-check count. */
|
||||
int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
|
||||
|
||||
/* #174/#219: assert a failing cache write aborts the mirror cleanly instead of
|
||||
crashing. Returns the failed-check count. */
|
||||
/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
|
||||
drop just the entry otherwise. Returns the failed-check count. */
|
||||
int cache_write_failure_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
/* Exercise the hts_cache_reconcile() generation policies on file fixtures
|
||||
under <dir>. Returns the failed-check count. */
|
||||
int cache_reconcile_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
|
||||
under <dir> and assert every case degrades to STATUSCODE_INVALID without
|
||||
tainting a sibling entry. */
|
||||
int cache_corruption_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -197,7 +197,6 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
|
||||
htsblk blkretour;
|
||||
|
||||
hts_init_htsblk(&blkretour);
|
||||
//memset(&blkretour, 0, sizeof(htsblk)); // effacer
|
||||
blkretour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx
|
||||
// Lire en têtes restants
|
||||
sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
|
||||
@@ -207,8 +206,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
|
||||
strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
|
||||
strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
|
||||
}
|
||||
// CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
|
||||
//strcatbuff(data,"\r\n");
|
||||
// CR/LF final de l'en tête inutile car déja placé via la ligne vide
|
||||
// juste au dessus
|
||||
if (blkretour.totalsize > 0) {
|
||||
int len = (int) min(blkretour.totalsize, 32000);
|
||||
int pos = (int) strlen(data);
|
||||
|
||||
@@ -841,71 +841,6 @@ static unsigned int nlz8(unsigned char x) {
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* Sample. */
|
||||
#if 0
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
int i;
|
||||
int hex = 0;
|
||||
|
||||
#define READ_INT(DEST) \
|
||||
( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1) \
|
||||
|| (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
|
||||
|
||||
for(i = 1 ; i < argc ; i++) {
|
||||
unsigned int uc, from, to;
|
||||
|
||||
if (strcmp(argv[i], "--hex") == 0) {
|
||||
hex = 1;
|
||||
}
|
||||
else if (strcmp(argv[i], "--decimal") == 0) {
|
||||
hex = 0;
|
||||
}
|
||||
else if (strcmp(argv[i], "--decode") == 0) {
|
||||
#define RD fgetc_unlocked(stdin)
|
||||
#define WR(C) do { \
|
||||
if (C != -1) { \
|
||||
printf("%04x\n", C); \
|
||||
} else if (!feof(stdin)) { \
|
||||
fprintf(stderr, "read error\n"); \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
} while(0)
|
||||
while(!feof(stdin)) {
|
||||
READ_UNICODE(RD, WR);
|
||||
}
|
||||
#undef RD
|
||||
#undef WR
|
||||
}
|
||||
else if (strcmp(argv[i], "-range") == 0
|
||||
&& i + 2 < argc
|
||||
&& (++i, 1)
|
||||
&& READ_INT(from)
|
||||
&& (++i, 1)
|
||||
&& READ_INT(to)
|
||||
) {
|
||||
unsigned int i;
|
||||
for(i = from ; i < to ; i++) {
|
||||
#define EM(C) fputc_unlocked(C, stdout)
|
||||
EMIT_UNICODE(i, EM);
|
||||
#undef EM
|
||||
}
|
||||
}
|
||||
else if (READ_INT(uc)) {
|
||||
#define EM(C) fputc_unlocked(C, stdout)
|
||||
EMIT_UNICODE(uc, EM);
|
||||
#undef EM
|
||||
}
|
||||
else {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/* IDNA helpers. */
|
||||
#undef ADD_BYTE
|
||||
#undef INCREASE_CAPA
|
||||
|
||||
@@ -54,10 +54,6 @@ Please visit our Website: http://www.httrack.com
|
||||
// extension par défaut pour fichiers n'en ayant pas
|
||||
#define DEFAULT_EXT ".html"
|
||||
#define DEFAULT_EXT_SHORT ".htm"
|
||||
// #define DEFAULT_BIN_EXT ".bin"
|
||||
// #define DEFAULT_BIN_EXT_SHORT ".bin"
|
||||
// #define DEFAULT_EXT ".txt"
|
||||
// #define DEFAULT_EXT_SHORT ".txt"
|
||||
|
||||
// éviter les /nul, /con..
|
||||
#define HTS_OVERRIDE_DOS_FOLDERS 1
|
||||
@@ -80,10 +76,6 @@ Please visit our Website: http://www.httrack.com
|
||||
// always direct-to-disk (0/1)
|
||||
#define HTS_DIRECTDISK_ALWAYS 1
|
||||
|
||||
// gérer une table de hachage?
|
||||
// REMOVED
|
||||
// #define HTS_HASH 1
|
||||
|
||||
// fast cache (build hash table)
|
||||
#define HTS_FAST_CACHE 1
|
||||
|
||||
@@ -97,27 +89,10 @@ Please visit our Website: http://www.httrack.com
|
||||
// always transform a '//' into a sigle '/'
|
||||
#define HTS_STRIP_DOUBLE_SLASH 0
|
||||
|
||||
// case-sensitive pour les dossiers et fichiers (0/1)
|
||||
// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
|
||||
// pas très utile..
|
||||
// ..et pas bcp respecté]
|
||||
// REMOVED
|
||||
// #define HTS_CASSE 0
|
||||
|
||||
// Un fichier ayant une taille différente du content-length doit il être annulé?
|
||||
// SEE opt.tolerant and opt.http10
|
||||
// #define HTS_CL_IS_FATAL 0
|
||||
|
||||
// une erreur supprime le fichier sur disque
|
||||
// (non fixé pour cause de retry)
|
||||
#define HTS_REMOVE_BAD_FILES 0
|
||||
|
||||
// en cas de Range: xx- donnant un Content-length: xx
|
||||
// alors skipper le fichier, considéré comme transmis
|
||||
// #define HTS_SKIP_FULL_RANGE 1
|
||||
|
||||
// nombre max de filtres que l'utilisateur peut fixer
|
||||
// #define HTS_FILTERSMAX 10000
|
||||
#define HTS_FILTERSINC 1000
|
||||
|
||||
// connect non bloquant? (poll sur write)
|
||||
|
||||
407
src/htscore.c
407
src/htscore.c
@@ -119,46 +119,93 @@ hts_log_print(opt, LOG_INFO, "engine: end"); \
|
||||
RUN_CALLBACK0(opt, end); \
|
||||
}
|
||||
|
||||
#define XH_extuninit do { \
|
||||
HTMLCHECK_UNINIT \
|
||||
hts_record_free(opt); \
|
||||
if (filters && filters[0]) { \
|
||||
freet(filters[0]); filters[0]=NULL; \
|
||||
} \
|
||||
if (filters) { \
|
||||
freet(filters); filters=NULL; \
|
||||
} \
|
||||
back_delete_all(opt,&cache,sback); \
|
||||
back_free(&sback); \
|
||||
checkrobots_free(&robots);\
|
||||
if (cache.use) { freet(cache.use); cache.use=NULL; } \
|
||||
if (cache.dat) { fclose(cache.dat); cache.dat=NULL; } \
|
||||
if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
|
||||
if (cache.zipOutput) { \
|
||||
zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
|
||||
cache.zipOutput = NULL; \
|
||||
} \
|
||||
if (cache.zipInput) { \
|
||||
unzClose(cache.zipInput); \
|
||||
cache.zipInput = NULL; \
|
||||
} \
|
||||
if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
|
||||
if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
|
||||
if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
|
||||
if (opt->log != NULL) fflush(opt->log); \
|
||||
if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
|
||||
if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
|
||||
if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
|
||||
if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
|
||||
if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
|
||||
if (cache_tests) { coucal_delete(&cache_tests); } \
|
||||
if (template_header) { freet(template_header); template_header=NULL; } \
|
||||
if (template_body) { freet(template_body); template_body=NULL; } \
|
||||
if (template_footer) { freet(template_footer); template_footer=NULL; } \
|
||||
hash_free(&hash); \
|
||||
clearCallbacks(&opt->state.callbacks); \
|
||||
/*structcheck_init(-1);*/ \
|
||||
} while(0)
|
||||
#define XH_extuninit \
|
||||
do { \
|
||||
HTMLCHECK_UNINIT \
|
||||
hts_record_free(opt); \
|
||||
if (filters && filters[0]) { \
|
||||
freet(filters[0]); \
|
||||
filters[0] = NULL; \
|
||||
} \
|
||||
if (filters) { \
|
||||
freet(filters); \
|
||||
filters = NULL; \
|
||||
} \
|
||||
back_delete_all(opt, &cache, sback); \
|
||||
back_free(&sback); \
|
||||
checkrobots_free(&robots); \
|
||||
if (cache.use) { \
|
||||
freet(cache.use); \
|
||||
cache.use = NULL; \
|
||||
} \
|
||||
if (cache.dat) { \
|
||||
fclose(cache.dat); \
|
||||
cache.dat = NULL; \
|
||||
} \
|
||||
if (cache.ndx) { \
|
||||
fclose(cache.ndx); \
|
||||
cache.ndx = NULL; \
|
||||
} \
|
||||
if (cache.zipOutput) { \
|
||||
zipClose(cache.zipOutput, \
|
||||
"Created by HTTrack Website Copier/" HTTRACK_VERSION); \
|
||||
cache.zipOutput = NULL; \
|
||||
} \
|
||||
if (cache.zipInput) { \
|
||||
unzClose(cache.zipInput); \
|
||||
cache.zipInput = NULL; \
|
||||
} \
|
||||
if (cache.olddat) { \
|
||||
fclose(cache.olddat); \
|
||||
cache.olddat = NULL; \
|
||||
} \
|
||||
if (cache.lst) { \
|
||||
fclose(cache.lst); \
|
||||
cache.lst = opt->state.strc.lst = NULL; \
|
||||
} \
|
||||
if (cache.txt) { \
|
||||
fclose(cache.txt); \
|
||||
cache.txt = NULL; \
|
||||
} \
|
||||
if (opt->log != NULL) \
|
||||
fflush(opt->log); \
|
||||
if (makestat_fp) { \
|
||||
fclose(makestat_fp); \
|
||||
makestat_fp = NULL; \
|
||||
} \
|
||||
if (maketrack_fp) { \
|
||||
fclose(maketrack_fp); \
|
||||
maketrack_fp = NULL; \
|
||||
} \
|
||||
if (opt->accept_cookie) \
|
||||
cookie_save(opt->cookie, \
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), \
|
||||
StringBuff(opt->path_log), "cookies.txt")); \
|
||||
if (makeindex_fp) { \
|
||||
fclose(makeindex_fp); \
|
||||
makeindex_fp = NULL; \
|
||||
} \
|
||||
if (cache_hashtable) { \
|
||||
coucal_delete(&cache_hashtable); \
|
||||
} \
|
||||
if (cache_tests) { \
|
||||
coucal_delete(&cache_tests); \
|
||||
} \
|
||||
if (template_header) { \
|
||||
freet(template_header); \
|
||||
template_header = NULL; \
|
||||
} \
|
||||
if (template_body) { \
|
||||
freet(template_body); \
|
||||
template_body = NULL; \
|
||||
} \
|
||||
if (template_footer) { \
|
||||
freet(template_footer); \
|
||||
template_footer = NULL; \
|
||||
} \
|
||||
hash_free(&hash); \
|
||||
clearCallbacks(&opt->state.callbacks); \
|
||||
} while (0)
|
||||
#define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)
|
||||
|
||||
struct lien_buffers {
|
||||
@@ -523,7 +570,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
hash_struct *const hashptr = &hash;
|
||||
t_cookie BIGSTK cookie; // gestion des cookies
|
||||
|
||||
//char* tab_alloc=NULL;
|
||||
int ptr; // pointeur actuel sur les liens
|
||||
|
||||
//
|
||||
@@ -534,15 +580,12 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// pour les stats, nombre de fichiers & octets écrits
|
||||
LLint stat_fragment = 0; // pour la fragmentation
|
||||
|
||||
//TStamp istat_timestart; // départ pour calcul instantanné
|
||||
//
|
||||
TStamp last_info_shell = 0;
|
||||
int info_shell = 0;
|
||||
|
||||
// filtres
|
||||
char **filters = NULL;
|
||||
|
||||
//int filter_max=0;
|
||||
int filptr = 0;
|
||||
|
||||
//
|
||||
@@ -582,7 +625,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// noter heure actuelle de départ en secondes
|
||||
memset(&HTS_STAT, 0, sizeof(HTS_STAT));
|
||||
HTS_STAT.stat_timestart = time_local();
|
||||
//istat_timestart=stat_timestart;
|
||||
HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
|
||||
/* reset stats */
|
||||
HTS_STAT.HTS_TOTAL_RECV = 0;
|
||||
@@ -616,9 +658,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// initialiser usercommand
|
||||
usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");
|
||||
|
||||
// initialiser structcheck
|
||||
// structcheck_init(1);
|
||||
|
||||
// initialiser verif_backblue
|
||||
verif_backblue(opt, NULL);
|
||||
verif_external(opt, 0, 0);
|
||||
@@ -638,9 +677,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
|
||||
"templates/index-footer.html"), HTS_INDEX_FOOTER);
|
||||
|
||||
// initialiser mimedefs
|
||||
//get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
|
||||
|
||||
// Initialiser indexation
|
||||
if (opt->kindex)
|
||||
index_init(StringBuff(opt->path_html));
|
||||
@@ -684,7 +720,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
opt->filters.filters = &filters;
|
||||
//
|
||||
opt->filters.filptr = &filptr;
|
||||
//opt->filters.filter_max=&filter_max;
|
||||
|
||||
// hash table
|
||||
opt->hash = &hash;
|
||||
@@ -731,8 +766,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
else if (*a == '-')
|
||||
joker = 1;
|
||||
|
||||
if (joker) { // joker ou filters
|
||||
//char* p;
|
||||
if (joker) { // joker ou filters
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
int type;
|
||||
int plus = 0;
|
||||
@@ -787,12 +821,11 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
XH_extuninit;
|
||||
return 0;
|
||||
}
|
||||
//opt->filters.filters=filters;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
} else { // adresse normale
|
||||
} else { // adresse normale
|
||||
char BIGSTK url[HTS_URLMAXSIZE * 2];
|
||||
|
||||
// prochaine adresse
|
||||
@@ -867,7 +900,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
htsbuff_cat(&primarybuff, "\n");
|
||||
}
|
||||
}
|
||||
// fclose(fp);
|
||||
hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
|
||||
StringBuff(opt->filelist));
|
||||
|
||||
@@ -922,7 +954,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
#endif
|
||||
|
||||
// backing
|
||||
//soc_max=opt->maxsoc;
|
||||
if (opt->maxsoc > 0) {
|
||||
#if BDEBUG==2
|
||||
_CLRSCR;
|
||||
@@ -1236,15 +1267,10 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// tester r.adr
|
||||
if (!error) {
|
||||
// erreur, pas de fichier chargé:
|
||||
if ((!r.adr) && (r.is_write == 0)
|
||||
&& (r.statuscode != 301)
|
||||
&& (r.statuscode != 302)
|
||||
&& (r.statuscode != 303)
|
||||
&& (r.statuscode != 307)
|
||||
&& (r.statuscode != 412)
|
||||
&& (r.statuscode != 416)
|
||||
) {
|
||||
// error=1;
|
||||
if ((!r.adr) && (r.is_write == 0) && (r.statuscode != 301) &&
|
||||
(r.statuscode != 302) && (r.statuscode != 303) &&
|
||||
(r.statuscode != 307) && (r.statuscode != 412) &&
|
||||
(r.statuscode != 416)) {
|
||||
|
||||
// peut être que le fichier était trop gros?
|
||||
if ((istoobig
|
||||
@@ -1326,30 +1352,8 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
||may_be_hypertext_mime(opt, r.contenttype, urlfil())) /* Is real media, .. */
|
||||
) {
|
||||
|
||||
/* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
|
||||
//if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
|
||||
// char *charset;
|
||||
// char *pos;
|
||||
// if (r.charset[0] != '\0') {
|
||||
// charset = strdup(r.charset);
|
||||
// } else {
|
||||
// charset = hts_getCharsetFromMeta(r.adr, r.size);
|
||||
// }
|
||||
// if (charset != NULL) {
|
||||
// char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
|
||||
// /* Use new buffer */
|
||||
// if (utf8 != NULL) {
|
||||
// freet(r.adr);
|
||||
// r.size = strlen(utf8);
|
||||
// r.adr = utf8;
|
||||
// /* New UTF-8 charset */
|
||||
// r.charset[0] = '\0';
|
||||
// strcpy(r.charset, "utf-8");
|
||||
// }
|
||||
// /* Free charset */
|
||||
// free(charset);
|
||||
// }
|
||||
//}
|
||||
/* Convert charset to UTF-8 - NOT! (what about links ? remote server
|
||||
* side will have troubles with converted names) */
|
||||
|
||||
/* Check bogus chars */
|
||||
if ((r.adr) && (r.size)) {
|
||||
@@ -1490,50 +1494,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
|
||||
}
|
||||
}
|
||||
// MOVED IN back_finalize()
|
||||
//
|
||||
// --------------------
|
||||
// REAL MEDIA HACK
|
||||
// Check if we have to load locally the file
|
||||
// --------------------
|
||||
//if (!error) {
|
||||
// if (r.statuscode == HTTP_OK) { // OK (ou 304 en backing)
|
||||
// if (r.adr==NULL) { // Written file
|
||||
// if (may_be_hypertext_mime(r.contenttype, urlfil())) { // to parse!
|
||||
// LLint sz;
|
||||
// sz=fsize_utf8(savename());
|
||||
// if (sz>0) { // ok, exists!
|
||||
// if (sz < 8192) { // ok, small file --> to parse!
|
||||
// FILE* fp=FOPEN(savename(),"rb");
|
||||
// if (fp) {
|
||||
// r.adr=malloct(sz + 1);
|
||||
// if (r.adr) {
|
||||
// if (fread(r.adr,1,sz,fp) == sz) {
|
||||
// r.size=sz;
|
||||
// r.adr[sz] = '\0';
|
||||
// r.is_write = 0;
|
||||
// } else {
|
||||
// freet(r.adr);
|
||||
// r.size=0;
|
||||
// r.adr = NULL;
|
||||
// r.statuscode=STATUSCODE_INVALID;
|
||||
// strcpybuff(r.msg, ".RAM read error");
|
||||
// }
|
||||
// fclose(fp);
|
||||
// fp=NULL;
|
||||
// // remove (temporary) file!
|
||||
// remove(savename());
|
||||
// }
|
||||
// if (fp)
|
||||
// fclose(fp);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
// EN OF REAL MEDIA HACK
|
||||
|
||||
// ---stockage en cache---
|
||||
// stocker dans le cache?
|
||||
@@ -1654,24 +1614,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
#endif
|
||||
|
||||
/* info: updated */
|
||||
/*
|
||||
if (ptr>0) {
|
||||
// "mis à jour"
|
||||
if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) { // page modifiée
|
||||
if (strnotempty(savename())) {
|
||||
HTS_STAT.stat_updated_files++;
|
||||
//if ((opt->debug>0) && (opt->log!=NULL)) {
|
||||
hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
|
||||
}
|
||||
} else {
|
||||
if (!store_errpage) {
|
||||
hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
// ------------------------------------------------------
|
||||
// traitement (parsing)
|
||||
// ------------------------------------------------------
|
||||
@@ -1717,13 +1659,8 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
free(charset);
|
||||
}
|
||||
/* Could not detect charset: could it be UTF-8 ? */
|
||||
/* No, we can not do that: browsers do not do it
|
||||
/* No, we can not do that: browsers do not do it
|
||||
(and it would break links). */
|
||||
//if (page_charset[0] == '\0') {
|
||||
// if (is_unicode_utf8(r.adr, r.size)) {
|
||||
// strcpy(page_charset, "utf-8");
|
||||
// }
|
||||
//}
|
||||
/* Could not detect charset */
|
||||
if (page_charset[0] == '\0') {
|
||||
hts_log_print(opt, LOG_INFO,
|
||||
@@ -1801,10 +1738,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
XH_uninit;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// I'll have to segment this part
|
||||
// #include "htsparse.c"
|
||||
|
||||
}
|
||||
}
|
||||
// Fin parsing HTML
|
||||
@@ -1900,7 +1833,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
HTS_STAT.stat_files++;
|
||||
HTS_STAT.stat_bytes+=r.size;
|
||||
*/
|
||||
//printf("ok......\n");
|
||||
} else {
|
||||
// Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
|
||||
// récursion nous en empêche
|
||||
@@ -1969,8 +1901,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
}
|
||||
}
|
||||
|
||||
//printf("extern=%s\n",r.contenttype);
|
||||
|
||||
// ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
|
||||
if (r.adr != NULL || r.size == 0) {
|
||||
file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
|
||||
@@ -2092,16 +2022,9 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// prochain pass2
|
||||
while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
|
||||
ptr++;
|
||||
|
||||
//printf("first link==%d\n");
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
// copy abort state if necessary from outside
|
||||
//if (!exit_xh && opt->state.exit_xh) {
|
||||
// exit_xh=opt->state.exit_xh;
|
||||
//}
|
||||
// a-t-on dépassé le quota?
|
||||
if (!back_checkmirror(opt)) {
|
||||
ptr = opt->lien_tot;
|
||||
@@ -2137,47 +2060,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
hts_log_print(opt, LOG_NOTICE,
|
||||
"No data seems to have been transferred during this session! : restoring previous one!");
|
||||
XH_uninit;
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/old.dat")))
|
||||
&&
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))) {
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"));
|
||||
}
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
opt->state.exit_xh = 2; /* interrupted (no connection detected) */
|
||||
return 1;
|
||||
}
|
||||
@@ -2298,13 +2181,11 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
char BIGSTK htstime[256];
|
||||
char BIGSTK infoupdated[256];
|
||||
|
||||
// int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
|
||||
LLint n =
|
||||
(LLint) (HTS_STAT.HTS_TOTAL_RECV /
|
||||
(max(1, time_local() - HTS_STAT.stat_timestart)));
|
||||
|
||||
sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
|
||||
//sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
|
||||
infoupdated[0] = '\0';
|
||||
if (opt->is_update) {
|
||||
if (HTS_STAT.stat_updated_files > 0) {
|
||||
@@ -2387,12 +2268,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
|
||||
*/
|
||||
int engine_stats(void) {
|
||||
#if 0
|
||||
static FILE *debug_fp = NULL; /* ok */
|
||||
|
||||
if (!debug_fp)
|
||||
debug_fp = fopen("esstat.txt", "wb");
|
||||
#endif
|
||||
HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
|
||||
HTS_STAT.nb = 0;
|
||||
if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
|
||||
@@ -2403,10 +2278,6 @@ int engine_stats(void) {
|
||||
if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
|
||||
TStamp dif;
|
||||
|
||||
#if 0
|
||||
fprintf(debug_fp, "set timer %d\n", i);
|
||||
fflush(debug_fp);
|
||||
#endif
|
||||
dif = cdif - HTS_STAT.istat_timestart[i];
|
||||
if ((TStamp) (dif / 1000) > 0) {
|
||||
LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
|
||||
@@ -2425,10 +2296,6 @@ int engine_stats(void) {
|
||||
// timer #0 resync timer #1 when reaching 1 second limit
|
||||
if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
|
||||
if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
|
||||
#if 0
|
||||
fprintf(debug_fp, "resync timer 1\n");
|
||||
fflush(debug_fp);
|
||||
#endif
|
||||
HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
|
||||
HTS_STAT.istat_timestart[1] = cdif;
|
||||
HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
|
||||
@@ -2449,7 +2316,6 @@ void host_ban(httrackp * opt, int ptr,
|
||||
lien_back *const back = sback->lnk;
|
||||
const int back_max = sback->count;
|
||||
|
||||
//int l;
|
||||
int i;
|
||||
|
||||
if (host[0] == '!')
|
||||
@@ -2506,9 +2372,7 @@ void host_ban(httrackp * opt, int ptr,
|
||||
}
|
||||
|
||||
// effacer liens
|
||||
//l=strlen(host);
|
||||
for(i = 0; i < opt->lien_tot; i++) {
|
||||
//if (heap(i)->adr_len==l) { // même taille de chaîne
|
||||
for (i = 0; i < opt->lien_tot; i++) {
|
||||
// Calcul de taille sécurisée
|
||||
if (heap(i)) {
|
||||
if (heap(i)->adr) {
|
||||
@@ -2547,7 +2411,6 @@ void host_ban(httrackp * opt, int ptr,
|
||||
"WARNING! HostCancel detected memory leaks [null at %d]",
|
||||
i);
|
||||
}
|
||||
//}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2892,6 +2755,9 @@ int check_fatal_io_errno(void) {
|
||||
#endif
|
||||
#ifdef EROFS
|
||||
case EROFS: /* Read-only file system */
|
||||
#endif
|
||||
#ifdef EDQUOT
|
||||
case EDQUOT: /* Disk quota exceeded */
|
||||
#endif
|
||||
return 1;
|
||||
break;
|
||||
@@ -2961,7 +2827,6 @@ FILE *fileappend(filenote_strc * strc, const char *s) {
|
||||
// noter lst
|
||||
filenote(strc, s, NULL);
|
||||
|
||||
// if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s); // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
|
||||
strcpybuff(fname, s);
|
||||
|
||||
#if HTS_DOSNAME
|
||||
@@ -3005,7 +2870,6 @@ int filecreateempty(filenote_strc * strc, const char *filename) {
|
||||
int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
|
||||
// gestion du fichier liste liste
|
||||
if (params) {
|
||||
//filecreate_params* p = (filecreate_params*) params;
|
||||
strcpybuff(strc->path, params->path);
|
||||
strc->lst = params->lst;
|
||||
return 0;
|
||||
@@ -3085,9 +2949,8 @@ void usercommand_exe(const char *cmd, const char *file) {
|
||||
}
|
||||
}
|
||||
|
||||
static void postprocess_file(httrackp * opt, const char *save, const char *adr,
|
||||
static void postprocess_file(httrackp *opt, const char *save, const char *adr,
|
||||
const char *fil) {
|
||||
//int first = 0;
|
||||
/* MIME-html archive to build */
|
||||
if (opt != NULL && opt->mimehtml) {
|
||||
if (adr != NULL && strcmp(adr, "primary") == 0) {
|
||||
@@ -3115,7 +2978,6 @@ static void postprocess_file(httrackp * opt, const char *save, const char *adr,
|
||||
}
|
||||
|
||||
if (!opt->state.mimehtml_created) {
|
||||
//first = 1;
|
||||
opt->state.mimefp =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
@@ -3259,28 +3121,6 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// vérifier taux de transfert
|
||||
#if 0
|
||||
void check_rate(TStamp stat_timestart, int maxrate) {
|
||||
// vérifier taux de transfert (pas trop grand?)
|
||||
/*
|
||||
if (maxrate>0) {
|
||||
int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart)); // taux actuel de transfert
|
||||
HTS_STAT.HTS_TOTAL_RECV_STATE=0;
|
||||
if (r>maxrate) { // taux>taux autorisé
|
||||
int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
|
||||
if (taux<15)
|
||||
HTS_STAT.HTS_TOTAL_RECV_STATE=1; // ralentir un peu (<15% dépassement)
|
||||
else if (taux<50)
|
||||
HTS_STAT.HTS_TOTAL_RECV_STATE=2; // beaucoup (<50% dépassement)
|
||||
else
|
||||
HTS_STAT.HTS_TOTAL_RECV_STATE=3; // énormément (>50% dépassement)
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
#endif
|
||||
|
||||
// ---
|
||||
// sous routines liées au moteur et au backing
|
||||
|
||||
@@ -3290,21 +3130,8 @@ int backlinks_done(const struct_back * sback,
|
||||
int ptr) {
|
||||
int n = 0;
|
||||
|
||||
#if 0
|
||||
int i;
|
||||
|
||||
//Links done and stored in cache
|
||||
for(i = ptr + 1; i < lien_tot; i++) {
|
||||
if (heap(i)) {
|
||||
if (heap(i)->pass2 == -1) {
|
||||
n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
// finalized in background
|
||||
n += HTS_STAT.stat_background;
|
||||
#endif
|
||||
n += back_done_incache(sback);
|
||||
return n;
|
||||
}
|
||||
@@ -3441,8 +3268,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
/* on a déja parcouru */
|
||||
if (p < cache->ptr_ant)
|
||||
p = cache->ptr_ant;
|
||||
while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
|
||||
//while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
|
||||
while (p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
|
||||
int ok = 1;
|
||||
|
||||
// on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
|
||||
@@ -3454,12 +3280,6 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
ok = 0;
|
||||
}
|
||||
|
||||
// Why in hell did I do that ?
|
||||
//if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0'
|
||||
// && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0) // lookup in liens_record
|
||||
//{
|
||||
// ok = 0;
|
||||
//}
|
||||
if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
|
||||
|| hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
|
||||
ok = 0;
|
||||
@@ -3488,7 +3308,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
}
|
||||
p++;
|
||||
} // while
|
||||
} // while
|
||||
/* sauver position dernière anticipation */
|
||||
cache->ptr_ant = p;
|
||||
cache->ptr_last = ptr;
|
||||
@@ -3750,20 +3570,7 @@ HTSEXT_API hts_boolean hts_has_stopped(httrackp *opt) {
|
||||
return ended;
|
||||
}
|
||||
|
||||
// régler en cours de route les paramètres réglables..
|
||||
// -1 : erreur
|
||||
//HTSEXT_API int hts_setopt(httrackp* set_opt) {
|
||||
// if (set_opt) {
|
||||
// httrackp* engine_opt=hts_declareoptbuffer(NULL);
|
||||
// if (engine_opt) {
|
||||
// //_hts_setopt=opt;
|
||||
// copy_htsopt(set_opt,engine_opt);
|
||||
// }
|
||||
// }
|
||||
// return 0;
|
||||
//}
|
||||
// ajout d'URL
|
||||
// -1 : erreur
|
||||
HTSEXT_API hts_boolean hts_addurl(httrackp *opt, char **url) {
|
||||
if (url)
|
||||
opt->state._hts_addurl = url;
|
||||
@@ -4024,10 +3831,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
|
||||
heap_top()->link_import = 0; // pas mode import
|
||||
|
||||
// écrire autres paramètres de la structure-lien
|
||||
//if (meme_adresse)
|
||||
heap_top()->premier = heap(ptr)->premier;
|
||||
//else // sinon l'objet père est le précédent lui même
|
||||
// heap_top()->premier=ptr;
|
||||
|
||||
heap_top()->precedent = ptr;
|
||||
// noter la priorité
|
||||
@@ -4038,9 +3842,6 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
|
||||
heap_top()->pass2 = max(pass_fix, numero_passe);
|
||||
heap_top()->retry = opt->retry;
|
||||
|
||||
//strcpybuff(heap_top()->adr,adr);
|
||||
//strcpybuff(heap_top()->fil,fil);
|
||||
//strcpybuff(heap_top()->sav,save);
|
||||
hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
|
||||
heap_top()->adr, heap_top()->fil,
|
||||
heap_top()->sav);
|
||||
|
||||
@@ -216,6 +216,7 @@ struct cache_back {
|
||||
int zipEntriesCapa;
|
||||
hts_boolean
|
||||
zipWriteFailed; /**< a cache write failed; stop touching the stream */
|
||||
int zipWriteFailures; /**< consecutive entry write failures; reset on store */
|
||||
};
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_hash_struct
|
||||
@@ -411,10 +412,6 @@ char *readfile_utf8(const char *fil);
|
||||
|
||||
char *readfile_or(const char *fil, const char *defaultdata);
|
||||
|
||||
#if 0
|
||||
void check_rate(TStamp stat_timestart, int maxrate);
|
||||
#endif
|
||||
|
||||
/* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
|
||||
Not thread-safe; call from the single crawl loop. */
|
||||
|
||||
@@ -466,11 +463,6 @@ int ask_continue(httrackp * opt);
|
||||
/* Number of decimal digits in n. */
|
||||
int nombre_digit(int n);
|
||||
|
||||
// Java
|
||||
#if 0
|
||||
int hts_add_file(char *file, int file_position);
|
||||
#endif
|
||||
|
||||
// Polling
|
||||
#if HTS_POLL
|
||||
int check_flot(T_SOC s);
|
||||
|
||||
@@ -145,7 +145,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char *url = NULL; // URLS séparées par un espace
|
||||
int url_sz = 65535;
|
||||
|
||||
//char url[65536]; // URLS séparées par un espace
|
||||
// the parametres
|
||||
int httrack_logmode = 3; // ONE log file
|
||||
|
||||
@@ -290,12 +289,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
if (tmp_argc == 1) { /* pas -P & co */
|
||||
if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
|
||||
if (argv_url < 0)
|
||||
argv_url = 0; // -1==force -> 1=one url already detected, wipe all previous options
|
||||
//if (argv_url>=0) {
|
||||
argv_url = 0; // -1==force -> 1=one url already detected, wipe all
|
||||
// previous options
|
||||
argv_url++;
|
||||
if (!argv_firsturl)
|
||||
argv_firsturl = x_argv[x_argc - 1];
|
||||
//}
|
||||
} else {
|
||||
if (strcmp(tmp_argv[0], "-h") == 0) {
|
||||
help(argv[0], !opt->quiet);
|
||||
@@ -324,8 +322,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
} else if (tmp_argc == 2) {
|
||||
if ((strcmp(tmp_argv[0], "-%L") == 0)) { // liste d'URLs
|
||||
if (argv_url < 0)
|
||||
argv_url = 0; // -1==force -> 1=one url already detected, wipe all previous options
|
||||
//if (argv_url>=0)
|
||||
argv_url = 0; // -1==force -> 1=one url already detected, wipe all
|
||||
// previous options
|
||||
argv_url++; /* forcer */
|
||||
}
|
||||
}
|
||||
@@ -403,15 +401,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
else
|
||||
inQuote = 1;
|
||||
} else if (!inQuote && !noDbl && argv[na][i] == ',') {
|
||||
//StringAddchar(path, '\0');
|
||||
//j = 0;
|
||||
path = &opt->path_log;
|
||||
} else {
|
||||
StringAddchar(*path, argv[na][i]);
|
||||
//path[j++] = argv[na][i];
|
||||
}
|
||||
}
|
||||
//path[j++] = '\0';
|
||||
if (StringLength(opt->path_log) == 0) {
|
||||
StringCopyS(opt->path_log, opt->path_html);
|
||||
}
|
||||
@@ -420,7 +414,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
if (check_path(&opt->path_html, argv_firsturl)) {
|
||||
opt->dir_topindex = 1; // rebuilt top index
|
||||
}
|
||||
//printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));
|
||||
}
|
||||
break;
|
||||
} // switch
|
||||
@@ -539,74 +532,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
insert_after++;
|
||||
}
|
||||
}
|
||||
} while(lastp != NULL);
|
||||
//fclose(fp);
|
||||
} while (lastp != NULL);
|
||||
}
|
||||
}
|
||||
|
||||
// Existence d'un cache - pas de new mais un old.. renommer
|
||||
// No new cache but an old one? promote it
|
||||
#if DEBUG_STEPS
|
||||
printf("Checking cache\n");
|
||||
#endif
|
||||
if (!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip"))) {
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
}
|
||||
} else
|
||||
if ((!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.dat")))
|
||||
||
|
||||
(!fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))) {
|
||||
if ((fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.dat")))
|
||||
&&
|
||||
(fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))) {
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
//remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
//rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
|
||||
}
|
||||
}
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
|
||||
/* Interrupted mirror detected */
|
||||
if (!opt->quiet) {
|
||||
@@ -774,24 +708,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
HTS_PANIC_PRINTF(s);
|
||||
htsmain_free();
|
||||
return -1;
|
||||
#else
|
||||
#if 0
|
||||
char _args[8][256];
|
||||
char *args[8];
|
||||
|
||||
printf("Cheking for updates...\n");
|
||||
strcpybuff(_args[0], argv[0]);
|
||||
strcpybuff(_args[1], "--get");
|
||||
sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
|
||||
strcpybuff(_args[3], "--quickinfo");
|
||||
args[0] = _args[0];
|
||||
args[1] = _args[1];
|
||||
args[2] = _args[2];
|
||||
args[3] = _args[3];
|
||||
args[4] = NULL;
|
||||
if (execvp(args[0], args) == -1) {
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
//
|
||||
@@ -809,103 +725,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
}
|
||||
|
||||
// Compter urls/jokers
|
||||
/*
|
||||
if (argv_url<=0) {
|
||||
int na;
|
||||
argv_url=0;
|
||||
for(na=1;na<argc;na++) {
|
||||
if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
|
||||
na++; // sauter nom de proxy
|
||||
} else if (!cmdl_opt(argv[na])) {
|
||||
argv_url++; // un de plus
|
||||
} else if (strcmp(argv[na],"-h")==0) {
|
||||
help(argv[0],!opt->quiet);
|
||||
htsmain_free();
|
||||
return 0;
|
||||
} else {
|
||||
if ((strchr(argv[na],'q')!=NULL))
|
||||
opt->quiet=1; // ne pas poser de questions! (nohup par exemple)
|
||||
if ((strchr(argv[na],'i')!=NULL)) { // doit.log!
|
||||
argv_url=0;
|
||||
na=argc;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
// Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
|
||||
// Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
|
||||
/*
|
||||
if (argv_url==0) {
|
||||
//if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
|
||||
if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) { // un cache est présent
|
||||
|
||||
x_argvblk=(char*) calloct(32768,1);
|
||||
|
||||
if (x_argvblk!=NULL) {
|
||||
FILE* fp;
|
||||
int x_argc;
|
||||
|
||||
//strcpybuff(x_argvblk,"httrack ");
|
||||
fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
|
||||
if (fp) {
|
||||
linput(fp,x_argvblk+strlen(x_argvblk),8192);
|
||||
fclose(fp); fp=NULL;
|
||||
}
|
||||
|
||||
// calculer arguments selon derniers arguments
|
||||
x_argv[0]=argv[0];
|
||||
x_argc=1;
|
||||
{
|
||||
char* p=x_argvblk;
|
||||
do {
|
||||
x_argv[x_argc++]=p;
|
||||
//p=strstr(p," ");
|
||||
// exemple de chaine: "echo \"test\"" c:\a "\$0"
|
||||
p=next_token(p,1); // prochain token
|
||||
if (p) {
|
||||
*p=0; // octet nul (tableau)
|
||||
p++;
|
||||
}
|
||||
} while(p!=NULL);
|
||||
}
|
||||
// recopier arguments actuels (pointeurs uniquement)
|
||||
{
|
||||
int na;
|
||||
for(na=1;na<argc;na++) {
|
||||
if (strcmp(argv[na],"-O") != 0) // SAUF le path!
|
||||
x_argv[x_argc++]=argv[na];
|
||||
else
|
||||
na++;
|
||||
}
|
||||
}
|
||||
argc=x_argc; // nouvel argc
|
||||
argv=x_argv; // nouvel argv
|
||||
}
|
||||
|
||||
}
|
||||
//}
|
||||
}
|
||||
*/
|
||||
|
||||
// Vérifier quiet
|
||||
/*
|
||||
{
|
||||
int na;
|
||||
for(na=1;na<argc;na++) {
|
||||
if (!cmdl_opt(argv[na])) {
|
||||
if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
|
||||
na++; // sauter nom de proxy
|
||||
} else {
|
||||
if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
|
||||
opt->quiet=1; // ne pas poser de questions! (nohup par exemple)
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
/* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
|
||||
here, ahead of the no-URL usage gate below, so they need no dummy URL. */
|
||||
{
|
||||
@@ -1012,8 +831,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
} else { // plus de 2 paramètres
|
||||
// un fichier log existe?
|
||||
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-in_progress.lock"))) { // fichier lock?
|
||||
//char s[32];
|
||||
StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"))) { // fichier lock?
|
||||
|
||||
opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
|
||||
if (opt->quiet == 0) {
|
||||
@@ -1044,11 +863,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
}
|
||||
}
|
||||
} else
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
|
||||
//char s[32];
|
||||
} else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "index.html"))) {
|
||||
opt->cache = HTS_CACHE_TEST_UPDATE;
|
||||
if (opt->quiet == 0) {
|
||||
if ((fexist
|
||||
@@ -1143,7 +959,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
//
|
||||
case 'g': // récupérer un (ou plusieurs) fichiers isolés
|
||||
opt->wizard = HTS_WIZARD_AUTO;
|
||||
//opt->wizard=0; // pas de wizard
|
||||
opt->cache = HTS_CACHE_NONE; // ni de cache
|
||||
opt->makeindex = 0; // ni d'index
|
||||
httrack_logmode = 1; // erreurs à l'écran
|
||||
@@ -1243,9 +1058,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
while(isdigit((unsigned char) *(com + 1)))
|
||||
com++;
|
||||
break;
|
||||
//
|
||||
//case 'A': opt->urlmode=1; break;
|
||||
//case 'R': opt->urlmode=2; break;
|
||||
case 'K':
|
||||
opt->urlmode = HTS_URLMODE_ABSOLUTE;
|
||||
if (isdigit((unsigned char) *(com + 1))) {
|
||||
@@ -1257,8 +1069,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
while(isdigit((unsigned char) *(com + 1)))
|
||||
com++;
|
||||
}
|
||||
//if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
|
||||
//
|
||||
case 'c':
|
||||
if (isdigit((unsigned char) *(com + 1))) {
|
||||
sscanf(com + 1, "%d", &opt->maxsoc);
|
||||
@@ -1746,7 +1556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
} else {
|
||||
char *a;
|
||||
|
||||
//char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
|
||||
for(a = argv[na]; *a != '\0'; a++) {
|
||||
if (*a == ';') { /* next one */
|
||||
StringAddchar(opt->mimedefs, '\n');
|
||||
@@ -2085,10 +1894,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
HTS_PANIC_PRINTF(s);
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
break;
|
||||
|
||||
//case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
|
||||
} break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -2332,15 +2138,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
return 0;
|
||||
break;
|
||||
case '~': /* internal lib test */
|
||||
HTS_PANIC_PRINTF
|
||||
("Option #~ is disabled for security reasons");
|
||||
//Disabled because choke on GCC 4.3 (toni from links2linux.de)
|
||||
//{
|
||||
// char thisIsATestYouShouldSeeAnError[12];
|
||||
// const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
|
||||
// strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
|
||||
// return 0;
|
||||
//}
|
||||
HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
|
||||
break;
|
||||
case 'f':
|
||||
opt->flush = 1;
|
||||
@@ -2442,7 +2240,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
opt->proxy.active = 1;
|
||||
// Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
|
||||
a = argv[na] + strlen(argv[na]) - 1;
|
||||
// a=strstr(argv[na],":"); // port
|
||||
while((a > argv[na]) && (*a != ':') && (*a != '@'))
|
||||
a--;
|
||||
if (*a == ':') { // un port est présent, <proxy>:port
|
||||
@@ -2547,116 +2344,14 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
#endif
|
||||
#endif
|
||||
|
||||
//printf("WARNING! This is *only* a beta-release of HTTrack\n");
|
||||
io_flush;
|
||||
|
||||
#if DEBUG_STEPS
|
||||
printf("Cache & log settings\n");
|
||||
#endif
|
||||
|
||||
// on utilise le cache..
|
||||
// en cas de présence des deux versions, garder la version la plus avancée,
|
||||
// cad la version contenant le plus de fichiers
|
||||
if (opt->cache) {
|
||||
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) { // problemes..
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")) < 32768) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) > 65536) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->
|
||||
path_log),
|
||||
"hts-cache/new.zip")))
|
||||
{
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))
|
||||
&&
|
||||
fexist(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"))) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"))
|
||||
&&
|
||||
fexist(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"))) {
|
||||
// switcher si new<32Ko et old>65Ko (tailles arbitraires) ?
|
||||
// ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre
|
||||
// alors l'ancien cache
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat")) < 32768) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")) > 65536) {
|
||||
if (fsize
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->
|
||||
path_log),
|
||||
"hts-cache/new.dat")))
|
||||
{
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"));
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx"));
|
||||
//} else { // ne rien faire
|
||||
// remove("hts-cache/old.dat");
|
||||
// remove("hts-cache/old.ndx");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// If both cache generations exist, keep the most complete one
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
// Débuggage des en têtes
|
||||
if (_DEBUG_HEAD) {
|
||||
ioinfo =
|
||||
@@ -2732,7 +2427,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
{
|
||||
FILE *fp = NULL;
|
||||
|
||||
//int n=0;
|
||||
char t[256];
|
||||
|
||||
time_local_rfc822(t); // faut bien que ca serve quelque part l'heure RFC1945 arf'
|
||||
@@ -2766,9 +2460,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
|
||||
strcpybuff(n_lock,
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-in_progress.lock"));
|
||||
//sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-in_progress.lock"));
|
||||
/*do {
|
||||
if (!n)
|
||||
sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
|
||||
@@ -2844,8 +2537,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
fprintf(fp, LF);
|
||||
fclose(fp);
|
||||
fp = NULL;
|
||||
//} else if (opt->debug>1) {
|
||||
// printf("! FileOpen error, \"%s\"\n",strerror(errno));
|
||||
}
|
||||
}
|
||||
// petit message dans le lock
|
||||
|
||||
@@ -102,9 +102,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
|
||||
// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
|
||||
// * = toute lettre
|
||||
// --?-- : spécifique à HTTrack et aux ?
|
||||
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
|
||||
int *size_flag) {
|
||||
//int err=0;
|
||||
HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
LLint *size, int *size_flag) {
|
||||
if (strnotempty(joker) == 0) { // fin de chaine joker
|
||||
if (strnotempty(chaine) == 0) // fin aussi pour la chaine: ok
|
||||
return chaine;
|
||||
@@ -236,9 +235,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
|
||||
for(j = (int) (unsigned char) joker[i];
|
||||
j <= (int) (unsigned char) joker[i + 2]; j++)
|
||||
pass[j] = 1;
|
||||
|
||||
}
|
||||
// else err=1;
|
||||
i += 3;
|
||||
} else { // 1 car, ex: *[ ]
|
||||
pass[(int) (unsigned char) joker[i]] = 1;
|
||||
@@ -262,7 +259,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
|
||||
for(i = 0; i < 256; i++)
|
||||
pass[i] = 1; // tout autoriser
|
||||
jmp = 1;
|
||||
////if (joker[2]==LEFT) jmp=3; // permet de recher *<crochet ouvrant>
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
135
src/htsftp.c
135
src/htsftp.c
@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
|
||||
#endif
|
||||
|
||||
// ftp mode passif
|
||||
// #if HTS_INET6==0
|
||||
#define FTP_PASV 1
|
||||
// #else
|
||||
// no passive mode for v6
|
||||
// #define FTP_PASV 0
|
||||
// #endif
|
||||
|
||||
#define FTP_DEBUG 0
|
||||
//#define FORK_DEBUG 0
|
||||
|
||||
#if USE_BEGINTHREAD
|
||||
|
||||
@@ -206,20 +200,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
{
|
||||
char *a;
|
||||
|
||||
#if 0
|
||||
a = back->url_fil + strlen(back->url_fil) - 1;
|
||||
while((a > back->url_fil) && (*a != '/'))
|
||||
a--;
|
||||
if (*a != '/') {
|
||||
a = NULL;
|
||||
}
|
||||
#else
|
||||
a = back->url_fil;
|
||||
#endif
|
||||
if (a != NULL && *a != '\0') {
|
||||
#if 0
|
||||
a++; // sauter /
|
||||
#endif
|
||||
ftp_filename = a;
|
||||
if (strnotempty(a)) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
@@ -243,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unexpected PORT error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
}
|
||||
@@ -277,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"Unable to get server's address: %s", error);
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_NON_FATAL;
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
@@ -290,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
|
||||
if (soc_ctl == INVALID_SOCKET) {
|
||||
strcpybuff(back->r.msg, "Unable to create a socket");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
|
||||
SOCaddr_initport(server, port);
|
||||
// server.sin_port = htons((unsigned short int) port);
|
||||
|
||||
// connexion (bloquante, on est en thread)
|
||||
strcpybuff(back->info, "connect");
|
||||
|
||||
if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
|
||||
strcpybuff(back->r.msg, "Unable to connect to the server");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
_HALT_FTP return 0;
|
||||
#ifdef _WIN32
|
||||
@@ -345,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
// ok
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "TYPE I error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#if 0
|
||||
// --CWD--
|
||||
char *a;
|
||||
|
||||
a = back->url_fil + strlen(back->url_fil) - 1;
|
||||
while((a > back->url_fil) && (*a != '/'))
|
||||
a--;
|
||||
if (*a == '/') { // ok repéré
|
||||
char BIGSTK target[1024];
|
||||
|
||||
target[0] = '\0';
|
||||
strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
|
||||
if (strnotempty(target) == 0)
|
||||
strcatbuff(target, "/");
|
||||
strcpybuff(back->info, "cwd");
|
||||
snprintf(line, sizeof(line), "CWD %s", target);
|
||||
send_line(soc_ctl, line);
|
||||
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
|
||||
_CHECK_HALT_FTP;
|
||||
if (line[0] == '2') {
|
||||
send_line(soc_ctl, "TYPE I");
|
||||
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
|
||||
_CHECK_HALT_FTP;
|
||||
if (line[0] == '2') {
|
||||
// ok..
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "TYPE I error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unexpected ftp error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#endif
|
||||
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
|
||||
@@ -469,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
// -- fin analyse de l'adresse IP et du port --
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
@@ -501,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
}
|
||||
@@ -613,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
deletesoc(soc_dat);
|
||||
soc_dat = INVALID_SOCKET;
|
||||
//
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
|
||||
linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"RETR command error: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
@@ -626,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
soc_dat = INVALID_SOCKET;
|
||||
//
|
||||
strcpybuff(back->r.msg, "Unable to connect");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unable to create a socket");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"Unable to resolve IP %s: %s", adr_ip, error);
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
#else
|
||||
@@ -663,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
//T_SOC soc_dat;
|
||||
if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
|
||||
strcpybuff(back->r.msg, "Unable to accept connection");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"RETR command error: %s", linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
@@ -683,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
#endif
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unable to listen to a port");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#endif
|
||||
@@ -707,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
int len = 1;
|
||||
int read_len = 1024;
|
||||
|
||||
//HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues
|
||||
|
||||
while((len > 0) && (!stop_ftp(back))) {
|
||||
// attendre les données
|
||||
len = 1; // pas d'erreur pour le moment
|
||||
switch (wait_socket_receive(soc_dat, timeout)) {
|
||||
case -1:
|
||||
strcpybuff(back->r.msg, "FTP read error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
len = 0; // fin
|
||||
break;
|
||||
case 0:
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
|
||||
timeout);
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
len = 0; // fin
|
||||
break;
|
||||
@@ -743,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
*/
|
||||
strcpybuff(back->r.msg, "Write error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
len = 0; // error
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unexpected write error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else { // Erreur ou terminé
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
} else { // Erreur ou terminé
|
||||
back->r.statuscode = 0;
|
||||
if (back->r.totalsize > 0
|
||||
&& back->r.size != back->r.totalsize) {
|
||||
@@ -762,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
}
|
||||
read_len = 1024;
|
||||
//HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues
|
||||
}
|
||||
}
|
||||
if (back->r.fp) {
|
||||
@@ -771,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unable to write file");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
@@ -787,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
|
||||
if (line[0] == '2') { // OK
|
||||
strcpybuff(back->r.msg, "OK");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = HTTP_OK;
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
|
||||
linejmp(line));
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "FTP read error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
}
|
||||
@@ -822,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
back->r.statuscode = HTTP_OK;
|
||||
strcpybuff(back->r.msg, "OK");
|
||||
}
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -989,9 +907,7 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
|
||||
break;
|
||||
}
|
||||
|
||||
//HTS_TOTAL_RECV_CHECK(dummy); // Diminuer au besoin si trop de données reçues
|
||||
switch (recv(soc, &b, 1, 0)) {
|
||||
//case 0: break; // pas encore --> erreur (on attend)!
|
||||
case 1:
|
||||
HTS_STAT.HTS_TOTAL_RECV += 1; // compter flux entrant
|
||||
if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
|
||||
@@ -1118,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
|
||||
int stop_ftp(lien_back * back) {
|
||||
if (back->stop_ftp) {
|
||||
strcpybuff(back->r.msg, "Cancelled by User");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
|
||||
#define str (buffers->str)
|
||||
#define argv (buffers->argv)
|
||||
|
||||
//char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
|
||||
//char *mainpath = (char *) malloct(256);
|
||||
//char *projname = (char *) malloct(256);
|
||||
//char *stropt = (char *) malloct(2048); // options
|
||||
//char *stropt2 = (char *) malloct(2048); // options longues
|
||||
//char *strwild = (char *) malloct(2048); // wildcards
|
||||
//char *cmd = (char *) malloct(4096);
|
||||
//char *str = (char *) malloct(256);
|
||||
//char **argv = (char **) malloct(256 * sizeof(char *));
|
||||
|
||||
//
|
||||
char *a;
|
||||
|
||||
@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
|
||||
strcatbuff(stropt2, str);
|
||||
strcatbuff(stropt2, projname);
|
||||
strcatbuff(stropt2, "\" ");
|
||||
// Créer si ce n'est fait un index.html 1er niveau
|
||||
make_empty_index(str);
|
||||
//
|
||||
printf("\n");
|
||||
printf("Enter URLs (separated by commas or blank spaces) :");
|
||||
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
|
||||
}
|
||||
hts_main(argc, argv);
|
||||
}
|
||||
//} else {
|
||||
// help("httrack",1);
|
||||
}
|
||||
|
||||
/* Free buffers */
|
||||
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
|
||||
printf("Unable to create a temporary proxy (no remaining port)\n");
|
||||
}
|
||||
|
||||
// Créer un index.html vide
|
||||
void make_empty_index(const char *str) {
|
||||
#if 0
|
||||
if (!fexist(fconcat(str, "index.html"))) {
|
||||
FILE *fp = fopen(fconcat(str, "index.html"), "wb");
|
||||
|
||||
if (fp) {
|
||||
fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
|
||||
fprintf(fp,
|
||||
"<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
|
||||
CRLF);
|
||||
fclose(fp);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// mini-aide (h: help)
|
||||
// y
|
||||
void help(const char *app, int more) {
|
||||
@@ -808,7 +777,4 @@ void help(const char *app, int more) {
|
||||
infomsg("[compiled: " HTS_PLATFORM_NAME "]");
|
||||
#endif
|
||||
infomsg(NULL);
|
||||
|
||||
// infomsg(" R *relative links (e.g ../link)\n");
|
||||
// infomsg(" A absolute links (e.g /www.adr/link)\n");
|
||||
}
|
||||
|
||||
@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;
|
||||
|
||||
void infomsg(const char *msg);
|
||||
void help(const char *app, int more);
|
||||
void make_empty_index(const char *str);
|
||||
void help_wizard(httrackp * opt);
|
||||
int help_query(const char *list, int def);
|
||||
void help_catchurl(const char *dest_path);
|
||||
|
||||
@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
char keyword[KEYW_LEN + 32];
|
||||
int i = 0;
|
||||
|
||||
//
|
||||
//int WordIndexSize = 1024;
|
||||
coucal WordIndexHash = NULL;
|
||||
FILE *tmpfp = NULL;
|
||||
|
||||
@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
|| (strfield2(mime, "text/css"))
|
||||
) {
|
||||
inscript = 1;
|
||||
//} else if (strfield2(mime, "text/vnd.wap.wml")) { // humm won't work in many cases
|
||||
// inscript=0;
|
||||
} else
|
||||
return 0;
|
||||
|
||||
@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
|
||||
// Process indexing for this page
|
||||
{
|
||||
//FILE* fp=NULL;
|
||||
//fp=fopen(concat(indexpath,"index.txt"),"ab");
|
||||
if (fp_tmpproject) {
|
||||
while(!feof(tmpfp)) {
|
||||
char line[KEYW_LEN + 32];
|
||||
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
intptr_t e = 0;
|
||||
|
||||
if (coucal_read(WordIndexHash, line, &e)) {
|
||||
//if (e) {
|
||||
char BIGSTK savelst[HTS_URLMAXSIZE * 2];
|
||||
|
||||
e++; /* 0 means "once" */
|
||||
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
fprintf(fp_tmpproject, "%s %d %s\n", line,
|
||||
(int) (KEYW_SORT_MAXCOUNT - e), savelst);
|
||||
hts_primindex_size++;
|
||||
//}
|
||||
}
|
||||
}
|
||||
}
|
||||
//fclose(fp);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
|
||||
off_t size = fpsize(fp_tmpproject);
|
||||
|
||||
if (size > 0) {
|
||||
//FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
|
||||
if (fp_tmpproject) {
|
||||
tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
|
||||
if (tab) {
|
||||
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
|
||||
if (total_hit) {
|
||||
if (mode == 1) // TEXT
|
||||
fprintf(fp, "\t=%d\r\n", total_hit);
|
||||
//else // HTML
|
||||
// fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
|
||||
if ((((total_hit * 1000) / hts_primindex_words) >=
|
||||
KEYW_USELESS1K)
|
||||
|| (((total_line * 1000) / index) >=
|
||||
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
|
||||
if (mode == 1) // TEXT
|
||||
fprintf(fp, "\t(%d)\r\n",
|
||||
((total_hit * 1000) / hts_primindex_words));
|
||||
//else // HTML
|
||||
// fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
|
||||
}
|
||||
}
|
||||
if (mode == 1) // TEXT
|
||||
|
||||
@@ -175,15 +175,14 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
#if JAVADEBUG
|
||||
printf("fopen\n");
|
||||
#endif
|
||||
if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
|
||||
//fprintf(stderr, "Cannot open input file.\n");
|
||||
if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
|
||||
NULL) {
|
||||
sprintf(str->err_msg, "Unable to open file %s", file);
|
||||
return 0; // une erreur..
|
||||
}
|
||||
#if JAVADEBUG
|
||||
printf("fread\n");
|
||||
#endif
|
||||
//if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) { // pas complet..
|
||||
if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
|
||||
fclose(fpout);
|
||||
sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
|
||||
@@ -257,12 +256,10 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
printf("addfiles\n");
|
||||
#endif
|
||||
{
|
||||
//unsigned int acess;
|
||||
unsigned int Class;
|
||||
unsigned int SClass;
|
||||
int i;
|
||||
|
||||
//acess = readshort(fpout);
|
||||
Class = readshort(fpout);
|
||||
SClass = readshort(fpout);
|
||||
|
||||
@@ -407,11 +404,9 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
|
||||
|
||||
p = &buffer[0];
|
||||
|
||||
//fflush(fp);
|
||||
trans.file_position = ftell(fp);
|
||||
length = readshort(fp);
|
||||
if (length < HTS_URLMAXSIZE) {
|
||||
// while ((length > 0) && (length<500)) {
|
||||
while(length > 0) {
|
||||
*p++ = fgetc(fp);
|
||||
|
||||
@@ -419,17 +414,13 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
|
||||
}
|
||||
*p = '\0';
|
||||
|
||||
//#if JDEBUG
|
||||
// if(tris(buffer)==1) printf("%s\n ",buffer);
|
||||
// if(tris(buffer)==2) printf("%s\n ",printname(buffer));
|
||||
//#endif
|
||||
if (tris(str->opt, buffer) == 1)
|
||||
str->addLink(str, buffer); /* trans.file_position */
|
||||
else if (tris(str->opt, buffer) == 2)
|
||||
str->addLink(str, printname(rname, buffer));
|
||||
|
||||
strcpy(trans.name, buffer);
|
||||
} else { // gros pb
|
||||
} else { // gros pb
|
||||
while((length > 0) && (!feof(fp))) {
|
||||
fgetc(fp);
|
||||
length--;
|
||||
@@ -445,7 +436,6 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
|
||||
}
|
||||
break;
|
||||
default:
|
||||
// printf("Type inconnue\n");
|
||||
// on arrête tout
|
||||
sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
|
||||
*error = 1;
|
||||
@@ -508,7 +498,6 @@ static char *printname(char rname[1024], char name[1024]) {
|
||||
return rname; // ""
|
||||
}
|
||||
p += 2;
|
||||
//rname=(char*)calloct(strlen(name)+8,sizeof(char));
|
||||
p1 = rname;
|
||||
for(j = 0; name[j] != '\0'; j++, p++) {
|
||||
if (*p == '/')
|
||||
|
||||
184
src/htslib.c
184
src/htslib.c
@@ -598,56 +598,27 @@ static const char *hts_mime_modern[][2] = {
|
||||
|
||||
// Reserved (RFC2396)
|
||||
#define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
|
||||
#define CHAR_RESERVED(c) ( CIS(c,';') \
|
||||
|| CIS(c,'/') \
|
||||
|| CIS(c,'?') \
|
||||
|| CIS(c,':') \
|
||||
|| CIS(c,'@') \
|
||||
|| CIS(c,'&') \
|
||||
|| CIS(c,'=') \
|
||||
|| CIS(c,'+') \
|
||||
|| CIS(c,'$') \
|
||||
|| CIS(c,',') )
|
||||
//#define CHAR_RESERVED(c) ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
|
||||
#define CHAR_RESERVED(c) \
|
||||
(CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') || \
|
||||
CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
|
||||
// Delimiters (RFC2396)
|
||||
#define CHAR_DELIM(c) ( CIS(c,'<') \
|
||||
|| CIS(c,'>') \
|
||||
|| CIS(c,'#') \
|
||||
|| CIS(c,'%') \
|
||||
|| CIS(c,'\"') )
|
||||
//#define CHAR_DELIM(c) ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
|
||||
#define CHAR_DELIM(c) \
|
||||
(CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
|
||||
// Unwise (RFC2396)
|
||||
#define CHAR_UNWISE(c) ( CIS(c,'{') \
|
||||
|| CIS(c,'}') \
|
||||
|| CIS(c,'|') \
|
||||
|| CIS(c,'\\') \
|
||||
|| CIS(c,'^') \
|
||||
|| CIS(c,'[') \
|
||||
|| CIS(c,']') \
|
||||
|| CIS(c,'`') )
|
||||
//#define CHAR_UNWISE(c) ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
|
||||
#define CHAR_UNWISE(c) \
|
||||
(CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
|
||||
CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
|
||||
// Special (escape chars) (RFC2396 + >127 )
|
||||
#define CHAR_LOW(c) ( ((unsigned char)(c) <= 31) )
|
||||
#define CHAR_HIG(c) ( ((unsigned char)(c) >= 127) )
|
||||
#define CHAR_SPECIAL(c) ( CHAR_LOW(c) || CHAR_HIG(c) )
|
||||
// We try to avoid them and encode them instead
|
||||
#define CHAR_XXAVOID(c) ( CIS(c,' ') \
|
||||
|| CIS(c,'*') \
|
||||
|| CIS(c,'\'') \
|
||||
|| CIS(c,'\"') \
|
||||
|| CIS(c,'&') \
|
||||
|| CIS(c,'!') )
|
||||
//#define CHAR_XXAVOID(c) ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
|
||||
#define CHAR_MARK(c) ( CIS(c,'-') \
|
||||
|| CIS(c,'_') \
|
||||
|| CIS(c,'.') \
|
||||
|| CIS(c,'!') \
|
||||
|| CIS(c,'~') \
|
||||
|| CIS(c,'*') \
|
||||
|| CIS(c,'\'') \
|
||||
|| CIS(c,'(') \
|
||||
|| CIS(c,')') )
|
||||
//#define CHAR_MARK(c) ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
|
||||
#define CHAR_XXAVOID(c) \
|
||||
(CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') || \
|
||||
CIS(c, '&') || CIS(c, '!'))
|
||||
#define CHAR_MARK(c) \
|
||||
(CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') || \
|
||||
CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))
|
||||
|
||||
// conversion éventuelle / vers antislash
|
||||
#ifdef _WIN32
|
||||
@@ -841,15 +812,12 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
// treat: traiter header?
|
||||
// waitconnect: attendre le connect()
|
||||
// note: dans retour, on met les params du proxy
|
||||
T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
|
||||
const char *xsend, const char *adr, const char *fil, htsblk * retour) {
|
||||
//htsblk retour;
|
||||
//int bufl=TAILLE_BUFFER; // 8Ko de buffer
|
||||
T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
|
||||
const char *xsend, const char *adr, const char *fil,
|
||||
htsblk *retour) {
|
||||
T_SOC soc = INVALID_SOCKET;
|
||||
char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];
|
||||
|
||||
//char *p,*q;
|
||||
|
||||
// retour prédéfini: erreur
|
||||
if (retour) {
|
||||
retour->adr = NULL;
|
||||
@@ -934,7 +902,6 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
|
||||
strcpybuff(retour->msg, "Unable to open local file");
|
||||
else {
|
||||
// Note: On passe par un FILE* (plus propre)
|
||||
//soc=open(fil,O_RDONLY,0); // en lecture seule!
|
||||
retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb"); // ouvrir
|
||||
if (retour->fp == NULL)
|
||||
@@ -1011,17 +978,10 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
|
||||
|
||||
} while(strnotempty(rcvd));
|
||||
|
||||
//rcvsize=-1; // forCER CHARGEMENT INCONNU
|
||||
|
||||
//if (retour)
|
||||
// retour->totalsize=rcvsize;
|
||||
|
||||
} else { // si GET, on recevra l'en tête APRES
|
||||
//rcvsize=-1; // on ne connait pas la taille de l'en-tête
|
||||
} else { // si GET, on recevra l'en tête APRES
|
||||
if (retour)
|
||||
retour->totalsize = -1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1118,19 +1078,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
char BIGSTK buffer_head_request[16384];
|
||||
buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };
|
||||
|
||||
//int use_11=0; // HTTP 1.1 utilisé
|
||||
int direct_url = 0; // ne pas analyser l'url (exemple: ftp://)
|
||||
const char *search_tag = NULL;
|
||||
|
||||
// Initialize buffer
|
||||
buffer_head_request[0] = '\0';
|
||||
|
||||
// header Date
|
||||
//strcatbuff(buff,"Date: ");
|
||||
//time_gmt_rfc822(buff); // obtenir l'heure au format rfc822
|
||||
//sendc("\n");
|
||||
//strcatbuff(buff,buff);
|
||||
|
||||
// possibilité non documentée: >post: et >postfile:
|
||||
// si présence d'un tag >post: alors executer un POST
|
||||
// exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
|
||||
@@ -1224,11 +1177,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
}
|
||||
|
||||
// protocole
|
||||
if (!retour->req.http11) { // forcer HTTP/1.0
|
||||
//use_11=0;
|
||||
if (!retour->req.http11) { // forcer HTTP/1.0
|
||||
print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
|
||||
} else { // Requète 1.1
|
||||
//use_11=1;
|
||||
} else { // Requète 1.1
|
||||
print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
|
||||
}
|
||||
|
||||
@@ -1364,9 +1315,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
//strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");
|
||||
|
||||
// Custom header(s)
|
||||
if (strnotempty(retour->req.headers)) {
|
||||
@@ -1520,14 +1469,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
a += strlen("filename=");
|
||||
while(is_space(*a))
|
||||
a++;
|
||||
//a=strchr(a,'"');
|
||||
if (a) {
|
||||
char *c = NULL;
|
||||
|
||||
//a++; /* jump " */
|
||||
while((c = strchr(a, '/'))) /* skip all / (see RFC2616) */
|
||||
a = c + 1;
|
||||
//b=strchr(a+1,'"');
|
||||
b = a + strlen(a) - 1;
|
||||
while(is_space(*b))
|
||||
b--;
|
||||
@@ -1544,16 +1490,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
} else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
|
||||
while(is_realspace(*(rcvd + p)))
|
||||
p++; // sauter espaces
|
||||
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
|
||||
//struct tm* tm_time=convert_time_rfc822(rcvd+p);
|
||||
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
|
||||
strcpybuff(retour->lastmodified, rcvd + p);
|
||||
}
|
||||
} else if ((p = strfield(rcvd, "Date:")) != 0) {
|
||||
if (strnotempty(retour->lastmodified) == 0) { /* pas encore de last-modified */
|
||||
while(is_realspace(*(rcvd + p)))
|
||||
p++; // sauter espaces
|
||||
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
|
||||
//struct tm* tm_time=convert_time_rfc822(rcvd+p);
|
||||
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
|
||||
strcpybuff(retour->lastmodified, rcvd + p);
|
||||
}
|
||||
}
|
||||
@@ -1566,14 +1510,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
else // erreur.. ignorer
|
||||
retour->etag[0] = '\0';
|
||||
}
|
||||
}
|
||||
// else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) { // chunk!
|
||||
else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
|
||||
} else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
|
||||
while(is_realspace(*(rcvd + p)))
|
||||
p++; // sauter espaces
|
||||
if (strfield(rcvd + p, "chunked")) {
|
||||
retour->is_chunk = 1; // chunked
|
||||
//retour->http11=2; // chunked
|
||||
retour->is_chunk = 1; // chunked
|
||||
#if HDEBUG
|
||||
printf("ok, Transfer-Encoding: détecté\n");
|
||||
#endif
|
||||
@@ -1750,7 +1691,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
retour->location[0] = '\0';
|
||||
}
|
||||
}
|
||||
} else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) { // ohh un cookie
|
||||
} else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
|
||||
(cookie)) { // ohh un cookie
|
||||
char *a = rcvd + p; // pointeur
|
||||
char domain[256]; // domaine cookie (.netscape.com)
|
||||
char path[256]; // chemin (/)
|
||||
@@ -1794,10 +1736,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
a++; // sauter espaces
|
||||
value_st = a;
|
||||
while((*a != ';') && (*a))
|
||||
a++; // prochain ;
|
||||
//while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++; // prochain " (et pas \")
|
||||
a++; // prochain ;
|
||||
value_end = a;
|
||||
//if (*a==';') { // finit par un ;
|
||||
// vérifier débordements
|
||||
if ((((int) (token_end - token_st)) < 200)
|
||||
&& (((int) (value_end - value_st)) < 8000)
|
||||
@@ -2139,8 +2079,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
|
||||
nl = READ_ERROR;
|
||||
}
|
||||
}
|
||||
//if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
|
||||
// nl=-1; // break
|
||||
|
||||
// libérer bloc tempo
|
||||
freet(buff);
|
||||
@@ -2228,9 +2166,6 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
|
||||
T_SOC soc;
|
||||
htsblk retour;
|
||||
|
||||
//int rcvsize=-1;
|
||||
//char* rcv=NULL; // adresse de retour
|
||||
//int bufl=TAILLE_BUFFER; // 8Ko de buffer
|
||||
TStamp tl;
|
||||
int timeout = 30; // timeout pour un check (arbitraire) // **
|
||||
|
||||
@@ -2239,11 +2174,8 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
|
||||
|
||||
loc[0] = '\0';
|
||||
hts_init_htsblk(&retour);
|
||||
//memset(&retour, 0, sizeof(htsblk)); // effacer
|
||||
retour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx
|
||||
|
||||
//soc=http_fopen(adr,fil,&retour,NULL); // ouvrir, + header
|
||||
|
||||
// on ouvre en head, et on traite l'en tête
|
||||
soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour); // ouvrir HEAD, + envoi header
|
||||
|
||||
@@ -2619,7 +2551,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
|
||||
|
||||
// chemin www... trop long!!
|
||||
if ((((int) (q - p))) > HTS_URLMAXSIZE) {
|
||||
//strcpybuff(retour.msg,"Path too long");
|
||||
return -1; // erreur
|
||||
}
|
||||
// recopier adrfil->adresse www..
|
||||
@@ -2805,7 +2736,6 @@ void deletesoc_r(htsblk * r) {
|
||||
#if HTS_USEOPENSSL
|
||||
if (r->ssl_con) {
|
||||
SSL_shutdown(r->ssl_con);
|
||||
// SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
|
||||
SSL_free(r->ssl_con);
|
||||
r->ssl_con = NULL;
|
||||
}
|
||||
@@ -3194,7 +3124,6 @@ int finput(T_SOC fd, char *s, int max) {
|
||||
int j = 0;
|
||||
|
||||
do {
|
||||
//c=fgetc(fp);
|
||||
if (read((int) fd, &c, 1) <= 0) {
|
||||
c = 0;
|
||||
}
|
||||
@@ -3210,7 +3139,7 @@ int finput(T_SOC fd, char *s, int max) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while((c != 0) && (j < max - 1));
|
||||
} while ((c != 0) && (j < max - 1));
|
||||
s[j] = '\0';
|
||||
return j;
|
||||
}
|
||||
@@ -4869,7 +4798,6 @@ LLint check_downloadable_bytes(int rate) {
|
||||
|
||||
time_now = mtime_local();
|
||||
elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
|
||||
// NO totally stupid - elapsed_useconds+=1000; // for the next second, too
|
||||
bytes_transferred_during_period =
|
||||
(HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);
|
||||
|
||||
@@ -4882,32 +4810,6 @@ LLint check_downloadable_bytes(int rate) {
|
||||
return TAILLE_BUFFER;
|
||||
}
|
||||
|
||||
//
|
||||
// 0 : OK
|
||||
// 1 : slow down
|
||||
#if 0
|
||||
int HTS_TOTAL_RECV_CHECK(int var) {
|
||||
if (HTS_STAT.HTS_TOTAL_RECV_STATE)
|
||||
return 1;
|
||||
/*
|
||||
{
|
||||
if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) {
|
||||
var = min(var,32);
|
||||
Sleep(250);
|
||||
} else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) {
|
||||
var = min(var,256);
|
||||
Sleep(100);
|
||||
} else {
|
||||
var/=2;
|
||||
if (var<=0) var=1;
|
||||
Sleep(50);
|
||||
}
|
||||
}
|
||||
*/
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
|
||||
// returns:
|
||||
// >0 : data received
|
||||
@@ -4916,7 +4818,6 @@ int HTS_TOTAL_RECV_CHECK(int var) {
|
||||
int hts_read(htsblk * r, char *buff, int size) {
|
||||
int retour;
|
||||
|
||||
// return read(soc,buff,size);
|
||||
if (r->is_file) {
|
||||
#if HTS_WIDE_DEBUG
|
||||
DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
|
||||
@@ -4934,7 +4835,6 @@ int hts_read(htsblk * r, char *buff, int size) {
|
||||
if (r->soc == INVALID_SOCKET)
|
||||
printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
|
||||
#endif
|
||||
//HTS_TOTAL_RECV_CHECK(size); // Diminuer au besoin si trop de données reçues
|
||||
#if HTS_USEOPENSSL
|
||||
if (r->ssl) {
|
||||
retour = SSL_read(r->ssl_con, buff, size);
|
||||
@@ -5369,18 +5269,13 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)
|
||||
|
||||
// --- Tracage des mallocs() ---
|
||||
#ifdef HTS_TRACE_MALLOC
|
||||
//#define htsLocker(A, N) htsLocker(A, N)
|
||||
#define htsLocker(A, N) do {} while(0)
|
||||
static mlink trmalloc = { NULL, 0, 0, NULL };
|
||||
|
||||
static int trmalloc_id = 0;
|
||||
static htsmutex *mallocMutex = NULL;
|
||||
static void hts_meminit(void) {
|
||||
//if (mallocMutex == NULL) {
|
||||
// mallocMutex = calloc(sizeof(*mallocMutex), 1);
|
||||
// htsLocker(mallocMutex, -999);
|
||||
//}
|
||||
}
|
||||
|
||||
static void hts_meminit(void) {}
|
||||
void *hts_malloc(size_t len) {
|
||||
void *adr;
|
||||
|
||||
@@ -5464,7 +5359,6 @@ void hts_free(void *adr) {
|
||||
htsboundary);
|
||||
lnk->next = lnk->next->next;
|
||||
free((void *) blk_free);
|
||||
//blk_free->id=-1;
|
||||
free((char *) adr - bsize);
|
||||
htsLocker(mallocMutex, 0);
|
||||
return;
|
||||
@@ -5533,7 +5427,6 @@ mlink *hts_find(char *adr) {
|
||||
|
||||
if (depl < 0)
|
||||
depl = -depl;
|
||||
//assertf(depl < 512000); /* near the stack frame.. doesn't look like malloc but stack variable */
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
@@ -5589,8 +5482,7 @@ int ftp_available(void) {
|
||||
}
|
||||
#else
|
||||
int ftp_available(void) {
|
||||
return 1; // ok!
|
||||
//return 0; // SOUS UNIX, PROBLEMESs
|
||||
return 1; // ok!
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -5804,7 +5696,6 @@ HTSEXT_API int hts_init(void) {
|
||||
assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
|
||||
}
|
||||
|
||||
// OpenSSL_add_all_algorithms();
|
||||
openssl_ctx = SSL_CTX_new(method);
|
||||
if (!openssl_ctx) {
|
||||
fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
|
||||
@@ -6126,7 +6017,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
opt->log = stdout;
|
||||
opt->errlog = stderr;
|
||||
opt->flush = HTS_TRUE;
|
||||
// opt->aff_progress=0;
|
||||
opt->keyboard = HTS_FALSE;
|
||||
//
|
||||
StringCopy(opt->path_html, "");
|
||||
@@ -6136,8 +6026,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
//
|
||||
opt->maxlink = 100000; // 100,000 liens max par défaut
|
||||
opt->maxfilter = 200; // 200 filtres max par défaut
|
||||
opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
|
||||
//opt->maxcache_anticipate=256; // maximum de liens à anticiper
|
||||
opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
|
||||
// PARAMETRABLE POUR L'INSTANT --
|
||||
opt->maxtime = -1; // temps max en secondes
|
||||
opt->maxrate = 100000; // taux maxi
|
||||
opt->maxconn = 5.0; // nombre connexions/s
|
||||
@@ -6290,11 +6180,9 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
|
||||
}
|
||||
|
||||
// defaut wrappers
|
||||
static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
|
||||
}
|
||||
static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
|
||||
// hts_freevar();
|
||||
}
|
||||
static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
|
||||
|
||||
static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
|
||||
static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -171,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
|
||||
// fonctions unix/winsock
|
||||
int hts_read(htsblk * r, char *buff, int size);
|
||||
|
||||
//int HTS_TOTAL_RECV_CHECK(int var);
|
||||
LLint check_downloadable_bytes(int rate);
|
||||
|
||||
HTSEXT_API int hts_uninit_module(void);
|
||||
@@ -190,7 +189,6 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
|
||||
int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
|
||||
const char *path, char *dst, size_t dst_size);
|
||||
|
||||
//int newhttp(char* iadr,char* err=NULL);
|
||||
T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
|
||||
int waitconnect);
|
||||
/* Like newhttp(), but connect to the addr_index-th resolved address of the host
|
||||
|
||||
@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
|
||||
|
||||
/* >>> Put all modules variables here */
|
||||
|
||||
#if 0
|
||||
t_gzopen gzopen = NULL;
|
||||
t_gzread gzread = NULL;
|
||||
t_gzclose gzclose = NULL;
|
||||
#endif
|
||||
|
||||
int V6_is_available = HTS_INET6;
|
||||
|
||||
static char WHAT_is_available[64] = "";
|
||||
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
|
||||
// while(chain->next != NULL) {
|
||||
// chain = chain->next;
|
||||
// }
|
||||
// chain->next = calloct(1, sizeof(htscallbacks));
|
||||
// assertf(chain->next != NULL);
|
||||
// chain = chain->next;
|
||||
// memset(chain, 0, sizeof(*chain));
|
||||
// chain->exitFnc = exitFnc;
|
||||
// chain->moduleHandle = moduleHandle;
|
||||
//}
|
||||
|
||||
void clearCallbacks(htscallbacks * chain_);
|
||||
void clearCallbacks(htscallbacks * chain_) {
|
||||
htscallbacks *chain;
|
||||
|
||||
@@ -1332,8 +1332,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
while((a > afs->save) && (*a != '.') && (*a != '/'))
|
||||
a--;
|
||||
if (*a != '.') { // agh pas de point
|
||||
//strcatbuff(save,".none"); // a éviter
|
||||
if (*a != '.') { // agh pas de point
|
||||
strcatbuff(afs->save, ".html"); // préférable!
|
||||
hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
|
||||
adr_complete, fil_complete, afs->save);
|
||||
@@ -1471,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
fil_simplifie(afs->save);
|
||||
|
||||
/* convert name to UTF-8 ? Note: already done while parsing. */
|
||||
//if (charset != NULL && charset[0] != '\0') {
|
||||
// char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
|
||||
|
||||
// if (s != NULL) {
|
||||
// hts_log_print(opt, LOG_DEBUG,
|
||||
// "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
|
||||
// save, s, charset);
|
||||
// strcpybuff(save, s);
|
||||
// free(s);
|
||||
// }
|
||||
//}
|
||||
|
||||
/* callback */
|
||||
RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
|
||||
@@ -1629,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
|
||||
int sameFil;
|
||||
|
||||
// NO - URL hack is only for stripping // and www.
|
||||
//if (opt->urlhack != 0)
|
||||
// sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
|
||||
//else
|
||||
sameFil = (strcmp(heap(i)->fil, normfil) == 0);
|
||||
if (sameAdr && sameFil) { // ok c'est le même lien, adresse déja définie
|
||||
/* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
|
||||
@@ -1698,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
strcpybuff(afs->save, tempo);
|
||||
|
||||
//printf("switched: %s\n",save);
|
||||
|
||||
} // if
|
||||
}
|
||||
#if DEBUG_SAVENAME
|
||||
printf("\nEnd search, %s\n", fil_complete);
|
||||
#endif
|
||||
} while(!nom_ok);
|
||||
|
||||
} while (!nom_ok);
|
||||
}
|
||||
//printf("'%s' %s %s\n",save,adr,fil);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -105,7 +105,6 @@ typedef struct htsfilters htsfilters;
|
||||
struct htsfilters {
|
||||
char ***filters; /**< pointer to the +/-pattern filter array */
|
||||
int *filptr; /**< pointer to the current filter count */
|
||||
// int* filter_max;
|
||||
};
|
||||
|
||||
/* User callbacks chain */
|
||||
@@ -440,7 +439,6 @@ struct httrackp {
|
||||
float maxconn; /**< max connections per second */
|
||||
int waittime; /**< scheduled start time (wall-clock seconds) */
|
||||
hts_cachemode cache; /**< cache generation mode */
|
||||
// int aff_progress; // progress bar
|
||||
hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
|
||||
t_proxy proxy; /**< proxy configuration */
|
||||
hts_savename_83
|
||||
@@ -492,7 +490,6 @@ struct httrackp {
|
||||
hts_verbosedisplay verbosedisplay; /**< animated text progress display */
|
||||
String footer; /**< footer/info line injected into pages */
|
||||
int maxcache; /**< in-memory cache backing limit (bytes) */
|
||||
// int maxcache_anticipate; // maximum links to anticipate (upper bound)
|
||||
hts_boolean ftp_proxy; /**< use the HTTP proxy for FTP too */
|
||||
String filelist; /**< file listing URLs to include */
|
||||
String urllist; /**< file listing filters to include */
|
||||
@@ -549,8 +546,6 @@ typedef struct hts_stat_struct hts_stat_struct;
|
||||
struct hts_stat_struct {
|
||||
LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
|
||||
LLint stat_bytes; /**< total bytes written to disk */
|
||||
// int HTS_TOTAL_RECV_STATE; // status: 0 ok 1: slow down a little 2: slow
|
||||
// down 3: a lot
|
||||
TStamp stat_timestart; /**< mirror start time */
|
||||
//
|
||||
LLint total_packed; /**< compressed bytes received (on the wire) */
|
||||
@@ -682,7 +677,6 @@ struct lien_url {
|
||||
int depth; /**< remaining allowed depth; >0 strong, 0 weak */
|
||||
int pass2; /**< second-pass marker; -1 means handled in background */
|
||||
char link_import; /**< imported after a move; skip the usual up/down rules */
|
||||
// int moved; // pointer to moved
|
||||
int retry; /**< remaining retries */
|
||||
int testmode; /**< test only: send just a HEAD */
|
||||
};
|
||||
@@ -727,9 +721,6 @@ struct lien_back {
|
||||
LLint chunk_size; /**< size of the chunk being loaded */
|
||||
LLint chunk_blocksize; /**< data size declared by the chunk */
|
||||
LLint compressed_size; /**< compressed size (stats only) */
|
||||
//
|
||||
// int links_index; // to access liens[links_index]
|
||||
//
|
||||
char info[256]; /**< status text, e.g. for FTP */
|
||||
int stop_ftp; /**< stop flag for FTP */
|
||||
int finalized; /**< finalized (memory optimization) */
|
||||
|
||||
343
src/htsparse.c
343
src/htsparse.c
@@ -192,6 +192,48 @@ static void hts_automate_lookup(const script_automate *aut) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Attribute name owning the quoted value at 'quote' inside a tag, spanning
|
||||
[name, *nend); NULL when the quote is not an attribute value. */
|
||||
static const char *dirty_attr_name(const char *quote, const char *tag_start,
|
||||
const char **nend) {
|
||||
const char *a = quote - 1;
|
||||
while (a > tag_start && is_taborspace(*a))
|
||||
a--;
|
||||
if (a == tag_start || *a != '=')
|
||||
return NULL;
|
||||
a--;
|
||||
while (a > tag_start && is_taborspace(*a))
|
||||
a--;
|
||||
*nend = a + 1;
|
||||
while (a > tag_start && *a != '=' && *a != '\"' && *a != '\'' &&
|
||||
!is_realspace(*a))
|
||||
a--;
|
||||
a++;
|
||||
// a name starting right after '<' is the tag name, not an attribute
|
||||
return a < *nend && a > tag_start + 1 ? a : NULL;
|
||||
}
|
||||
|
||||
/* Accept the in-tag quoted value at 'quote' for dirty parsing? Resolves the
|
||||
owning attribute itself (intag_startattr is unreliable mid-tag) and rejects
|
||||
no-detect/xmlns names. */
|
||||
static hts_boolean dirty_attr_detectable(const char *quote,
|
||||
const char *tag_start) {
|
||||
const char *nend;
|
||||
const char *name = dirty_attr_name(quote, tag_start, &nend);
|
||||
int i;
|
||||
if (name == NULL)
|
||||
return HTS_FALSE;
|
||||
for (i = 0; strnotempty(hts_nodetect[i]); i++) {
|
||||
const int l = strfield(name, hts_nodetect[i]);
|
||||
if (l && name + l == nend)
|
||||
return HTS_FALSE;
|
||||
}
|
||||
i = strfield(name, "xmlns");
|
||||
if (i && (name + i == nend || name[i] == ':'))
|
||||
return HTS_FALSE;
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
|
||||
static void hts_automate_increment(const script_automate *aut, int steps) {
|
||||
while (steps > 0) {
|
||||
@@ -353,16 +395,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// terminaison (" ou ') du "<body onLoad=.."
|
||||
int inscriptgen = 0; // on est dans un code générant, ex après obj.write("..
|
||||
|
||||
//int inscript_check_comments=0, inscript_in_comments=0; // javascript comments
|
||||
char scriptgen_q = '\0'; // caractère faisant office de guillemet (' ou ")
|
||||
|
||||
//int no_esc_utf=0; // ne pas echapper chars > 127
|
||||
int nofollow = 0; // ne pas scanner
|
||||
|
||||
//
|
||||
int parseall_lastc = '\0'; // dernier caractère parsé pour parseall
|
||||
|
||||
//int parseall_incomment=0; // dans un /* */ (exemple: a = /* URL */ "img.gif";)
|
||||
//
|
||||
const char *intag_start = html;
|
||||
const char *intag_name = NULL;
|
||||
@@ -475,12 +514,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
lastsaved = html;
|
||||
}
|
||||
}
|
||||
// Detect UTF8 format
|
||||
//if (is_unicode_utf8(r->adr, (unsigned int) r->size) == 1) {
|
||||
// no_esc_utf=1;
|
||||
//} else {
|
||||
// no_esc_utf=0;
|
||||
//}
|
||||
|
||||
// Hack to prevent any problems with ram files of other files
|
||||
*(r->adr + r->size) = '\0';
|
||||
@@ -645,8 +678,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
) {
|
||||
intag = 1;
|
||||
intag_ctype = 0;
|
||||
//parseall_incomment=0;
|
||||
//inquote=0; // effacer quote
|
||||
intag_start = html;
|
||||
for(intag_name = html + 1; is_realspace(*intag_name); intag_name++) ;
|
||||
intag_start_valid = 1;
|
||||
@@ -664,8 +695,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (len > 0) {
|
||||
if (strfield(token, "content-type")) {
|
||||
intag_ctype = 1;
|
||||
//NOPE-we do not convert the whole page actually
|
||||
//intag_start[1] = 'X';
|
||||
} else if (strfield(token, "refresh")) {
|
||||
intag_ctype = 2;
|
||||
}
|
||||
@@ -726,7 +755,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil)), gmttime,
|
||||
HTTRACK_VERSIONID, /* EOF */ NULL);
|
||||
strcatbuff(tempo, eol);
|
||||
//fwrite(tempo,1,strlen(tempo),fp);
|
||||
HT_ADD(tempo);
|
||||
}
|
||||
// Emit charset ?
|
||||
@@ -754,6 +782,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
) {
|
||||
if (inscript_tag) {
|
||||
inscript_tag = inscript = 0;
|
||||
// reset the automaton on exit or its state leaks into plain HTML
|
||||
inscript_state_pos = INSCRIPT_START;
|
||||
intag = 0;
|
||||
incomment = 0;
|
||||
intag_start_valid = 0;
|
||||
@@ -765,8 +795,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
intag = 0; //inquote=0;
|
||||
|
||||
// entrée dans du javascript?
|
||||
// on parse ICI car il se peut qu'on ait eu a parser les src=.. dedans
|
||||
//if (!inscript) { // sinon on est dans un obj.write("..
|
||||
// on parse ICI car il se peut qu'on ait eu a parser les src=..
|
||||
// dedans
|
||||
if ((intag_start_valid) && (check_tag(intag_start, "script")
|
||||
|| check_tag(intag_start, "style")
|
||||
)
|
||||
@@ -812,12 +842,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
#endif
|
||||
}
|
||||
//}
|
||||
}
|
||||
//else if (*adr==34) {
|
||||
// inquote=(inquote?0:1);
|
||||
//}
|
||||
else if (intag || inscript || in_media) { // nous sommes dans un tag/commentaire, tester si on recoit un tag
|
||||
} else if (intag || inscript ||
|
||||
in_media) { // nous sommes dans un tag/commentaire, tester si
|
||||
// on recoit un tag
|
||||
int p_type = 0;
|
||||
int p_nocatch = 0;
|
||||
int p_searchMETAURL = 0; // chercher ..URL=<url>
|
||||
@@ -837,6 +864,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (*html == inscript_tag_lastc) {
|
||||
/* sortir */
|
||||
inscript_tag = inscript = 0;
|
||||
inscript_state_pos = INSCRIPT_START;
|
||||
incomment = 0;
|
||||
if (opt->parsedebug) {
|
||||
HT_ADD("<@@ /inscript @@>");
|
||||
@@ -1055,27 +1083,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (p) {
|
||||
if (intag_ctype == 1) {
|
||||
p = 0;
|
||||
#if 0
|
||||
//if ((pos=rech_tageq(html, "content"))) {
|
||||
char temp[256];
|
||||
char *token = NULL;
|
||||
int len = rech_endtoken(html + pos, &token);
|
||||
|
||||
if (len > 0 && len < sizeof(temp) - 2) {
|
||||
char *chpos;
|
||||
|
||||
temp[0] = '\0';
|
||||
strncat(temp, token, len);
|
||||
if ((chpos = strstr(temp, "charset"))
|
||||
&& (chpos = strchr(chpos, '='))
|
||||
) {
|
||||
chpos++;
|
||||
while(is_space(*chpos))
|
||||
chpod++;
|
||||
//chpos
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
// <META HTTP-EQUIV="Refresh" CONTENT="3;URL=http://www.example.com">
|
||||
else if (intag_ctype == 2) {
|
||||
@@ -1260,21 +1267,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
} else if (inscript) {
|
||||
|
||||
#if 0
|
||||
/* Check // javascript comments */
|
||||
if (*html == 10 || *html == 13) {
|
||||
inscript_check_comments = 1;
|
||||
inscript_in_comments = 0;
|
||||
} else if (inscript_check_comments) {
|
||||
if (!is_realspace(*html)) {
|
||||
inscript_check_comments = 0;
|
||||
if (html[0] == '/' && html[1] == '/') {
|
||||
inscript_in_comments = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Parse */
|
||||
assertf(inscript_name != NULL);
|
||||
if (*html == '/'
|
||||
@@ -1287,18 +1279,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
&& inscript_locked == 0) {
|
||||
const char *a = html;
|
||||
|
||||
//while(is_realspace(*(--a)));
|
||||
while(is_realspace(*a))
|
||||
a--;
|
||||
a--;
|
||||
if (*a == '<') { // sûr que c'est un tag?
|
||||
inscript = 0;
|
||||
inscript_state_pos = INSCRIPT_START;
|
||||
if (opt->parsedebug) {
|
||||
HT_ADD("<@@ /inscript @@>");
|
||||
}
|
||||
}
|
||||
} else if (inscript_state_pos ==
|
||||
INSCRIPT_START /*!inscript_in_comments */ ) {
|
||||
} else if (inscript_state_pos == INSCRIPT_START) {
|
||||
/*
|
||||
Script Analyzing - different types supported:
|
||||
foo="url"
|
||||
@@ -1487,12 +1478,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
|
||||
} else { // ptr == 0
|
||||
//p=rech_tageq(adr,"primary"); // lien primaire, yeah
|
||||
p = 0; // No stupid tag anymore, raw link
|
||||
valid_p = 1; // Valid even if p==0
|
||||
while((html[p] == '\r') || (html[p] == '\n'))
|
||||
p++;
|
||||
//can_avoid_quotes=1;
|
||||
ending_p = '\r';
|
||||
}
|
||||
|
||||
@@ -1505,25 +1494,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// risque: générer de faux fichiers parazites
|
||||
// fix: ne parse plus dans les commentaires
|
||||
// ------------------------------------------------------------
|
||||
if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 && (ptr > 0) && (!in_media) /* && (!inscript_in_comments) */ ) { // option parsing "brut"
|
||||
//int incomment_justquit=0;
|
||||
if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 &&
|
||||
(ptr > 0) && (!in_media)) { // option parsing "brut"
|
||||
if (!is_realspace(*html)) {
|
||||
int noparse = 0;
|
||||
|
||||
// Gestion des /* */
|
||||
#if 0
|
||||
if (inscript) {
|
||||
if (parseall_incomment) {
|
||||
if ((*html == '/') && (*(html - 1) == '*'))
|
||||
parseall_incomment = 0;
|
||||
incomment_justquit = 1; // ne pas noter dernier caractère
|
||||
} else {
|
||||
if ((*html == '/') && (*(html + 1) == '*'))
|
||||
parseall_incomment = 1;
|
||||
}
|
||||
} else
|
||||
parseall_incomment = 0;
|
||||
#endif
|
||||
/* ensure automate state 0 (not in comments, quotes..) */
|
||||
if (inscript
|
||||
&& (inscript_state_pos != INSCRIPT_INQUOTE
|
||||
@@ -1537,7 +1512,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
|
||||
// recherche d'URLs
|
||||
if (!noparse) {
|
||||
//if ((!parseall_incomment) && (!noparse)) {
|
||||
if (!p) { // non déja trouvé
|
||||
if (html != r->adr) { // >1 caractère
|
||||
// scanner les chaines
|
||||
@@ -1561,15 +1535,16 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
&& (count > 0)) {
|
||||
char c;
|
||||
|
||||
//char* aend;
|
||||
//
|
||||
//aend=a; // sauver début
|
||||
a++;
|
||||
while(is_taborspace(*a))
|
||||
a++;
|
||||
c = *a;
|
||||
if (strchr("),;>/+\r\n", c)) { // exemple: ..img.gif";
|
||||
// le / est pour funct("img.gif" /* URL */);
|
||||
// in-tag, an attribute value ends at its quote: no
|
||||
// delimiter required after it (mid-tag attrs, #201)
|
||||
if (strchr("),;>/+\r\n", c) ||
|
||||
(intag && !inscript && intag_start_valid &&
|
||||
dirty_attr_detectable(html, intag_start))) {
|
||||
// '/' covers a value followed by a JS comment
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
char type[256];
|
||||
int url_ok = 0; // url valide?
|
||||
@@ -1614,10 +1589,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (!invalid_url) {
|
||||
// Un plus à la fin? Alors ne pas prendre sauf si extension ("/toto.html#"+tag)
|
||||
if (c != '+') { // PAS de plus à la fin
|
||||
#if 0
|
||||
char *a;
|
||||
#endif
|
||||
// "Comparisons of scheme names MUST be case-insensitive" (RFC2616)
|
||||
// "Comparisons of scheme names MUST be
|
||||
// case-insensitive" (RFC2616)
|
||||
if ((strfield(tempo, "http:"))
|
||||
|| (strfield(tempo, "ftp:"))
|
||||
#if HTS_USEOPENSSL
|
||||
@@ -1630,15 +1603,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (inscript) // sinon si pas javascript, méfiance (répertoire style base?)
|
||||
url_ok = 1;
|
||||
}
|
||||
#if 0
|
||||
else if ((a = strchr(tempo, '/'))) { // un slash: ok..
|
||||
if (inscript) { // sinon si pas javascript, méfiance (style "text/css")
|
||||
if (strchr(a + 1, '/')) // un seul / : abandon (STYLE type='text/css')
|
||||
if (!strchr(tempo, ' ')) // avoid spaces (too dangerous for comments)
|
||||
url_ok = 1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
// Prendre si extension reconnue
|
||||
if (!url_ok) {
|
||||
@@ -1717,9 +1681,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
}
|
||||
}
|
||||
} // p == 0
|
||||
} // p == 0
|
||||
|
||||
} // not in comment
|
||||
} // not in comment
|
||||
|
||||
// plus dans un commentaire
|
||||
if (inscript_state_pos == INSCRIPT_START
|
||||
@@ -1728,17 +1692,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
|
||||
} // if realspace
|
||||
} // if parseall
|
||||
} // if parseall
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// p!=0 : on a repéré un éventuel lien
|
||||
// ------------------------------------------------------------
|
||||
//
|
||||
if ((p > 0) || (valid_p)) { // on a repéré un lien
|
||||
//int lien_valide=0;
|
||||
if ((p > 0) || (valid_p)) { // on a repéré un lien
|
||||
const char *eadr = NULL; /* fin de l'URL */
|
||||
|
||||
//char* quote_adr=NULL; /* adresse du ? dans l'adresse */
|
||||
int ok = 1;
|
||||
char quote = '\0';
|
||||
int quoteinscript = 0;
|
||||
@@ -1761,7 +1723,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
lastsaved = html; // dernier écrit+1
|
||||
}
|
||||
// sauter espaces
|
||||
// adr+=p;
|
||||
hts_automate_increment(&saut, p);
|
||||
while((is_space(*html)
|
||||
|| (inscriptgen && html[0] == '\\' && is_space(html[1])
|
||||
@@ -1790,8 +1751,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// sauter éventuel \" ou \' javascript
|
||||
if (inscript) { // on est dans un obj.write("..
|
||||
if (*html == '\\') {
|
||||
if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
|
||||
// html+=2; // sauter
|
||||
if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
|
||||
hts_automate_increment(&saut, 2);
|
||||
}
|
||||
}
|
||||
@@ -1866,7 +1826,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
ok = -1; // ne pas traiter ce lien
|
||||
|
||||
if (ok > 0) {
|
||||
//if (*eadr!=' ') {
|
||||
if (is_space(*eadr)) { // guillemets,CR, etc
|
||||
if ((*eadr == quote && (!quoteinscript || *(eadr - 1) == '\\')) // end quote
|
||||
|| (noquote && (*eadr == '\"' || *eadr == '\'')) // end at any quote
|
||||
@@ -1912,7 +1871,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
//}
|
||||
}
|
||||
eadr++;
|
||||
} while(ok == 1);
|
||||
@@ -1941,14 +1899,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
char BIGSTK lien[HTS_URLMAXSIZE * 2];
|
||||
int meme_adresse = 0; // 0 par défaut pour primary
|
||||
|
||||
//char *copie_de_adr=html;
|
||||
//char* p;
|
||||
|
||||
// construire lien (découpage)
|
||||
if (eadr - html - 1 < HTS_URLMAXSIZE) { // pas trop long?
|
||||
strncpy(lien, html, eadr - html - 1);
|
||||
lien[eadr - html - 1] = '\0';
|
||||
//printf("link: %s\n",lien);
|
||||
// supprimer les espaces
|
||||
while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
|
||||
lien[strlen(lien) - 1] = '\0';
|
||||
@@ -2192,7 +2146,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
a += 2;
|
||||
else
|
||||
a = lien;
|
||||
// while((*a) && (*a!='/') && (*a!=':')) a++;
|
||||
a = jump_toport(a);
|
||||
if (a) { // port
|
||||
int port = 0;
|
||||
@@ -2200,9 +2153,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
char *b = a + 1;
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
// FIXME
|
||||
//if (strfield(adr, "https:")) {
|
||||
//}
|
||||
#endif
|
||||
while(isdigit((unsigned char) *b)) {
|
||||
port *= 10;
|
||||
@@ -2293,13 +2243,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
*(a + 1) = '\0';
|
||||
}
|
||||
}
|
||||
//char BIGSTK tempo[HTS_URLMAXSIZE*2];
|
||||
//strcpybuff(tempo,"http://");
|
||||
//strcatbuff(tempo,urladr()); // host
|
||||
//if (*lien!='/')
|
||||
// strcatbuff(tempo,"/");
|
||||
//strcatbuff(tempo,lien);
|
||||
//strcpybuff(lien,tempo);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2328,22 +2271,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
// stocker base ou codebase?
|
||||
switch (p_type) {
|
||||
case 2:{
|
||||
//if (*lien!='/') strcatbuff(base,"/");
|
||||
strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
|
||||
}
|
||||
break; // base
|
||||
case -2:{
|
||||
//if (*lien!='/') strcatbuff(codebase,"/");
|
||||
strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
|
||||
}
|
||||
break; // base
|
||||
case 2: {
|
||||
strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
|
||||
} break; // base
|
||||
case -2: {
|
||||
strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
|
||||
} break; // base
|
||||
}
|
||||
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"code/codebase link %s base %s", lien,
|
||||
base);
|
||||
//printf("base code: %s - %s\n",lien,base);
|
||||
}
|
||||
|
||||
} else {
|
||||
@@ -2456,7 +2394,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// Tester si un lien doit être accepté ou refusé (wizard)
|
||||
// forbidden_url=1 : lien refusé
|
||||
// forbidden_url=0 : lien accepté
|
||||
//if ((ptr>0) && (p_type!=2) && (p_type!=-2)) { // tester autorisations?
|
||||
if ((p_type != 2) && (p_type != -2)) { // tester autorisations?
|
||||
if (!p_nocatch) {
|
||||
if (afs.af.adr[0] != '\0') {
|
||||
@@ -2506,9 +2443,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
|
||||
/* Calc */
|
||||
last_adr[0] = '\0';
|
||||
//char last_fil[HTS_URLMAXSIZE*2]="";
|
||||
strcpybuff(last_adr, afs.af.adr); // ancienne adresse
|
||||
//strcpybuff(last_fil,fil); // ancien chemin
|
||||
strcpybuff(last_adr, afs.af.adr); // ancienne adresse
|
||||
r_sv =
|
||||
url_savename(&afs, &former, heap(ptr)->adr, heap(ptr)->fil, opt,
|
||||
sback, cache, hash, ptr,
|
||||
@@ -2539,7 +2474,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
}
|
||||
}
|
||||
//import_done=1; // c'est un import!
|
||||
meme_adresse = 0; // on a changé
|
||||
}
|
||||
} else {
|
||||
@@ -2737,11 +2671,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (lienrelatif(tempo, sizeof(tempo), save,
|
||||
relativesavename()) == 0) {
|
||||
/* Never escape high-chars (we don't know the encoding!!) */
|
||||
inplace_escape_uri_utf(tempo, sizeof(tempo)); // escape with %xx
|
||||
//if (!no_esc_utf)
|
||||
// escape_uri(tempo); // escape with %xx
|
||||
//else
|
||||
// escape_uri_utf(tempo); // escape with %xx
|
||||
inplace_escape_uri_utf(
|
||||
tempo, sizeof(tempo)); // escape with %xx
|
||||
HT_ADD_HTMLESCAPED(tempo); // page externe
|
||||
if (add_url) {
|
||||
HT_ADD("?link="); // page externe
|
||||
@@ -2978,16 +2909,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// unquoted url() (CSS/JS): keep parens escaped
|
||||
if (ending_p == ')')
|
||||
escape_url_parens(tempo, sizeof(tempo));
|
||||
|
||||
//if (!no_esc_utf)
|
||||
// escape_uri(tempo); // escape with %xx
|
||||
//else {
|
||||
// /* No escaping at all - remaining upper chars will be escaped below */
|
||||
// /* FIXME - Should be done in all local cases */
|
||||
// //x_escape_html(tempo);
|
||||
// //escape_uri_utf(tempo); // FIXME - escape with %xx
|
||||
// //escape_uri(tempo); // escape with %xx
|
||||
//}
|
||||
}
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"relative link at %s build with %s and %s: %s",
|
||||
@@ -3045,22 +2966,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
HT_ADD(tempo4); // refresh code="
|
||||
}
|
||||
}
|
||||
//lastsaved=adr; // dernier écrit+1
|
||||
}
|
||||
|
||||
if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
|
||||
// convert to local codepage - NOT, already converted into %NN, and passed to the remote server so we do not have anything to do
|
||||
//if (str->page_charset_ != NULL && *str->page_charset_ != '\0') {
|
||||
// char *const local_save = hts_convertStringFromUTF8(tempo, strlen(tempo), str->page_charset_);
|
||||
// if (local_save != NULL) {
|
||||
// strcpybuff(tempo, local_save);
|
||||
// free(local_save);
|
||||
// } else {
|
||||
// if ((opt->debug>1) && (opt->log!=NULL)) {
|
||||
// fprintf(opt->log, "Warning: could not build local charset representation of '%s' in '%s'"LF, tempo, str->page_charset_);
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
// écrire le lien modifié, relatif
|
||||
// Note: escape all chars, even >127 (no UTF)
|
||||
@@ -3084,16 +2992,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
} // sinon le lien sera écrit normalement
|
||||
|
||||
#if 0
|
||||
if (fexist(save)) { // le fichier existe..
|
||||
adr[0] = '\0';
|
||||
//if ((opt->debug>0) && (opt->log!=NULL)) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Link has already been written on disk, cancelled: %s",
|
||||
save);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Security check */
|
||||
if (strlen(afs.save) >= HTS_URLMAXSIZE) {
|
||||
afs.af.adr[0] = '\0';
|
||||
@@ -3135,9 +3033,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
/*
|
||||
if (strnotempty(save)) {
|
||||
if (ishtml(opt,save) == 1) {
|
||||
// descore_prio = 2;
|
||||
} else {
|
||||
// descore_prio = 1;
|
||||
}
|
||||
}
|
||||
*/
|
||||
@@ -3173,8 +3069,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// >>>> CREER LE LIEN <<<<
|
||||
//
|
||||
// enregistrer lien à charger
|
||||
//heap_top()->adr[0]=heap_top()->fil[0]=heap_top()->sav[0]='\0';
|
||||
// même adresse: l'objet père est l'objet père de l'actuel
|
||||
|
||||
// DEBUT ROBOTS.TXT AJOUT
|
||||
if (!just_test_it) {
|
||||
@@ -3253,7 +3147,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
heap_top()->premier = heap(ptr)->premier;
|
||||
else // sinon l'objet père est le précédent lui même
|
||||
heap_top()->premier = heap_top_index();
|
||||
// heap_top()->premier=ptr;
|
||||
|
||||
heap_top()->precedent = ptr;
|
||||
// noter la priorité
|
||||
@@ -3265,9 +3158,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
heap_top()->pass2 = pass_fix;
|
||||
heap_top()->retry = opt->retry;
|
||||
|
||||
//strcpybuff(heap_top()->adr,adr);
|
||||
//strcpybuff(heap_top()->fil,fil);
|
||||
//strcpybuff(heap_top()->sav,save);
|
||||
if (!just_test_it) {
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"OK, NOTE: %s%s -> %s",
|
||||
@@ -3302,7 +3192,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (eadr > html) {
|
||||
hts_automate_increment(&saut, (int) (eadr - 1 - html));
|
||||
}
|
||||
// adr=eadr-1; // ** sauter
|
||||
|
||||
/* srcset candidate loop: skip the descriptor and comma, then
|
||||
re-enter the capture for the next URL. Backward goto, not a loop:
|
||||
@@ -3332,9 +3221,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
inscript_state_pos = INSCRIPT_START;
|
||||
} */
|
||||
|
||||
} // if (p)
|
||||
} // if (p)
|
||||
|
||||
} // si '<' ou '>'
|
||||
} // si '<' ou '>'
|
||||
|
||||
// plus loin
|
||||
html++; // automate will be checked next loop
|
||||
@@ -3404,9 +3293,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
*stre->exit_xh_ = 1; // exit requested
|
||||
XH_uninit;
|
||||
return -1;
|
||||
//adr = r->adr + r->size; // exit
|
||||
} else if (opt->state._hts_cancel == 1) {
|
||||
// adr = r->adr + r->size; // exit
|
||||
nofollow = 1; // moins violent
|
||||
opt->state._hts_cancel = 0;
|
||||
}
|
||||
@@ -3464,9 +3351,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
fp = NULL;
|
||||
}
|
||||
}
|
||||
// sauver fichier
|
||||
//structcheck(savename());
|
||||
//filesave(opt,r->adr,r->size,savename());
|
||||
|
||||
} // analyse OK
|
||||
|
||||
@@ -3505,17 +3389,11 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
// DEBUT rattrapage des 301,302,307..
|
||||
// ------------------------------------------------------------
|
||||
if (!error) {
|
||||
////////{
|
||||
// on a chargé un fichier en plus
|
||||
// if (!error) stat_loaded+=r.size;
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing
|
||||
// ------------------------------------------------------------
|
||||
if (HTTP_IS_REDIRECT(r->statuscode)) {
|
||||
//if (r->adr!=NULL) { // adr==null si fichier direct. [catch: davename normalement si cgi]
|
||||
//int i=0;
|
||||
// char* p;
|
||||
|
||||
hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, urladr(), urlfil());
|
||||
|
||||
@@ -3542,7 +3420,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
// A same-file alias redirect must be followed, not stubbed (#159).
|
||||
const hts_boolean same_savefile = hts_redirect_same_savefile(
|
||||
opt, urladr(), urlfil(), moved->adr, moved->fil);
|
||||
//if (ident_url_absolute(mov_url,moved->adr,moved->fil)!=-1) { // ok URL reconnue
|
||||
// c'est (en gros) la même URL..
|
||||
// si c'est un problème de casse dans le host c'est que le serveur est buggé
|
||||
// ("RFC says.." : host name IS case insensitive)
|
||||
@@ -3582,7 +3459,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
} /* sinon traité normalement */
|
||||
}
|
||||
|
||||
//if ((strfield2(moved->adr,urladr())!=0) && (strfield2(moved->fil,urlfil())!=0)) { // identique à casse près
|
||||
if (get_it == 1) {
|
||||
// court-circuiter le reste du traitement
|
||||
// et reculer pour mieux sauter
|
||||
@@ -3593,9 +3469,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
error = 1;
|
||||
hts_invalidate_link(opt, ptr); // invalidate hashtable entry
|
||||
// noter NOUVEAU lien
|
||||
//xxc xxc
|
||||
// set_prio_to=0+1; // protection if the moved URL is an html page!!
|
||||
//xxc xxc
|
||||
{
|
||||
// calculer lien et éventuellement modifier addresse/fichier
|
||||
if (url_savename(&savedmoved, NULL,
|
||||
@@ -3633,8 +3506,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
}
|
||||
}
|
||||
|
||||
//printf("-> %s %s %s\n",liens[lien_tot-1]->adr,liens[lien_tot-1]->fil,liens[lien_tot-1]->sav);
|
||||
|
||||
// note métaphysique: il se peut qu'il y ait un index.html et un INDEX.HTML
|
||||
// sous DOS ca marche pas très bien... mais comme je suis génial url_savename()
|
||||
// est à même de régler ce problème
|
||||
@@ -3756,7 +3627,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
|
||||
// cas où l'on peut reessayer
|
||||
switch (r->statuscode) {
|
||||
//case -1: can_retry=1; break;
|
||||
case STATUSCODE_TIMEOUT:
|
||||
if (opt->hostcontrol) { // timeout et retry épuisés
|
||||
if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) &&
|
||||
@@ -3888,7 +3758,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
// FIN rattrapage des 301,302,307..
|
||||
// ------------------------------------------------------------
|
||||
|
||||
} // if !error
|
||||
} // if !error
|
||||
|
||||
/* Apply changes */
|
||||
ENGINE_SAVE_CONTEXT();
|
||||
@@ -3955,9 +3825,6 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
|
||||
}
|
||||
}
|
||||
}
|
||||
// On désalloue le buffer d'enregistrement des chemins créée, au cas où pendant la pause
|
||||
// l'utilisateur ferait un rm -r après avoir effectué un tar
|
||||
// structcheck_init(1);
|
||||
{
|
||||
FILE *fp =
|
||||
fopen(fconcat
|
||||
@@ -4338,33 +4205,8 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
|
||||
"link #%d is ready, no more on the stack, skipping: %s%s..",
|
||||
ptr, urladr(), urlfil());
|
||||
|
||||
// prochain lien
|
||||
// ptr++;
|
||||
|
||||
return 2; // goto jump_if_done;
|
||||
|
||||
return 2; // goto jump_if_done;
|
||||
}
|
||||
#if 0
|
||||
/* FIXME - finalized HAS NO MORE THIS MEANING */
|
||||
/* link put in cache by the backing system for memory spare - reclaim */
|
||||
else if (back[b].finalized) {
|
||||
assertf(back[b].r.adr == NULL);
|
||||
/* read file in cache */
|
||||
back[b].r =
|
||||
cache_read_ro(opt, cache, back[b].url_adr, back[b].url_fil,
|
||||
back[b].url_sav, back[b].location_buffer);
|
||||
/* ensure correct location buffer set */
|
||||
back[b].r.location = back[b].location_buffer;
|
||||
if (back[b].r.statuscode == STATUSCODE_INVALID) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Unexpected error: %s%s not found anymore in cache",
|
||||
back[b].url_adr, back[b].url_fil);
|
||||
} else {
|
||||
hts_log_print(opt, LOG_DEBUG, "reclaim file %s%s (%d)", back[b].url_adr,
|
||||
back[b].url_fil, back[b].r.statuscode);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!opt->verbosedisplay) {
|
||||
if (!opt->quiet) {
|
||||
@@ -4388,7 +4230,6 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
//}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Vérificateur d'intégrité
|
||||
@@ -4556,9 +4397,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) {
|
||||
lien_back delayed_back;
|
||||
|
||||
//char BIGSTK delayed_ctype[128];
|
||||
// delayed_ctype[0] = '\0';
|
||||
// strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1); // copier content-type
|
||||
back_copy_static(&back[b], &delayed_back);
|
||||
|
||||
/* Delete entry */
|
||||
@@ -4566,6 +4404,12 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
back_maydelete(opt, cache, sback, b); // cancel
|
||||
b = -1;
|
||||
|
||||
/* the cancel may leave the now-unreferenced placeholder on disk
|
||||
* (#483) */
|
||||
if (fexist_utf8(delayed_back.url_sav)) {
|
||||
back_delayed_discard(opt, &delayed_back);
|
||||
}
|
||||
|
||||
/* Recompute filename with MIME type */
|
||||
afs->save[0] = '\0';
|
||||
url_savename(afs, former, heap(ptr)->adr,
|
||||
@@ -4657,9 +4501,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
if (b >= 0) {
|
||||
lien_back delayed_back;
|
||||
|
||||
//char BIGSTK delayed_ctype[128];
|
||||
//delayed_ctype[0] = '\0';
|
||||
//strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1); // copier content-type
|
||||
back_copy_static(&back[b], &delayed_back);
|
||||
|
||||
/* Error */
|
||||
@@ -4782,6 +4623,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
|
||||
/* Still have a back reference */
|
||||
if (b >= 0) {
|
||||
/* move a still-writing placeholder before the url_sav patch
|
||||
blinds every cleanup to it (#483) */
|
||||
back_delayed_rename(opt, &back[b], afs->save);
|
||||
/* patch url_sav BEFORE finalize: it records/caches under this name
|
||||
*/
|
||||
strcpybuff(back[b].url_sav, afs->save);
|
||||
@@ -4808,13 +4652,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
/* 'no error page' selected or file discarded by size rules! */
|
||||
if (!opt->errpage || (in_error == STATUSCODE_TOO_BIG)) {
|
||||
/* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
|
||||
#if 0
|
||||
/* No (3.43) - don't do that. We must not post-exclude an authorized link, because this will prevent the cache
|
||||
system from processing it, leading to refetch it endlessly. Just accept it, and handle the error as
|
||||
usual during parsing.
|
||||
*/
|
||||
*forbidden_url = 1; /* Forbidden! */
|
||||
#endif
|
||||
/* Do not post-exclude the link here (*forbidden_url): the cache system
|
||||
would never process it again, and it would be refetched endlessly */
|
||||
if (in_error == STATUSCODE_TOO_BIG) {
|
||||
hts_log_print(opt, LOG_INFO,
|
||||
"link not taken because of its size (%d bytes) at %s%s",
|
||||
|
||||
@@ -66,7 +66,6 @@ struct htsmoduleStructExtended {
|
||||
char ***filters_;
|
||||
robots_wizard *robots_;
|
||||
hash_struct *hash_;
|
||||
//int *lien_max_;
|
||||
|
||||
/* Base & codebase */
|
||||
char *base;
|
||||
|
||||
@@ -1347,6 +1347,30 @@ static int st_cache_writefail(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
|
||||
int err;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "cache-corrupt: needs a directory\n");
|
||||
return 1;
|
||||
}
|
||||
err = cache_corruption_selftest(opt, argv[0]);
|
||||
printf("cache-corrupt: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_reconcile(httrackp *opt, int argc, char **argv) {
|
||||
int err;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "reconcile: needs a directory\n");
|
||||
return 1;
|
||||
}
|
||||
err = cache_reconcile_selftest(opt, argv[0]);
|
||||
printf("cache-reconcile: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
static int st_dns(httrackp *opt, int argc, char **argv) {
|
||||
const int err = dns_selftests(opt);
|
||||
|
||||
@@ -2119,6 +2143,10 @@ static const struct selftest_entry {
|
||||
st_cache_golden},
|
||||
{"cache-writefail", "<dir>", "cache write-failure handling self-test",
|
||||
st_cache_writefail},
|
||||
{"reconcile", "<dir>", "cache generation reconcile policy self-test",
|
||||
st_reconcile},
|
||||
{"cache-corrupt", "<dir>", "cache read-side corruption self-test",
|
||||
st_cache_corrupt},
|
||||
{"dns", "", "DNS resolver/cache self-test", st_dns},
|
||||
{"cookies", "", "cookie request-header self-test", st_cookies},
|
||||
{"useragent", "", "default User-Agent self-test", st_useragent},
|
||||
|
||||
@@ -265,7 +265,6 @@ T_SOC smallserver_init(int *port, char *adr) {
|
||||
SOCaddr_initport(server, *port);
|
||||
if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
|
||||
if (listen(soc, 10) >= 0) {
|
||||
// SOCaddr_inetntoa(adr, 128, server2);
|
||||
strcpy(adr, h_loc);
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
@@ -439,7 +438,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
meth = 10;
|
||||
} else {
|
||||
#ifdef _DEBUG
|
||||
// assert(FALSE);
|
||||
#endif
|
||||
}
|
||||
if (meth) {
|
||||
@@ -1360,7 +1358,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
|
||||
StringCat(headers, error_hdr);
|
||||
StringCat(output, error);
|
||||
//assert(file == NULL);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1388,12 +1385,10 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
!= StringLength(output)))
|
||||
) {
|
||||
#ifdef _DEBUG
|
||||
//assert(FALSE);
|
||||
#endif
|
||||
}
|
||||
} else {
|
||||
#ifdef _DEBUG
|
||||
// assert(FALSE);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -1540,8 +1535,7 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
|
||||
} while(strnotempty(test));
|
||||
}
|
||||
|
||||
if (!strnotempty(test)) { // éviter doublons
|
||||
// conv_printf(key,key);
|
||||
if (!strnotempty(test)) { // éviter doublons
|
||||
const size_t len = strlen(intkey);
|
||||
char *const buff = (char *) malloc(len + 1);
|
||||
|
||||
@@ -1632,8 +1626,6 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
|
||||
intkey = "";
|
||||
} else {
|
||||
if (loops > 0) {
|
||||
//err_msg += intkey;
|
||||
//err_msg += " ";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1740,13 +1732,7 @@ static void LANG_DELETE(void) {
|
||||
}
|
||||
|
||||
// sélection de la langue
|
||||
static void LANG_INIT(const char *path) {
|
||||
//CWinApp* pApp = AfxGetApp();
|
||||
//if (pApp) {
|
||||
/* pApp->GetProfileInt("Language","IntId",0); */
|
||||
LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
|
||||
//}
|
||||
}
|
||||
static void LANG_INIT(const char *path) { LANG_T(path, 0); }
|
||||
|
||||
static int LANG_T(const char *path, int l) {
|
||||
if (l >= 0) {
|
||||
|
||||
@@ -68,16 +68,6 @@ struct find_handle_struct {
|
||||
char path[2048];
|
||||
};
|
||||
#endif
|
||||
//#ifndef HTS_DEF_FWSTRUCT_topindex_chain
|
||||
//#define HTS_DEF_FWSTRUCT_topindex_chain
|
||||
//typedef struct topindex_chain topindex_chain;
|
||||
//#endif
|
||||
//struct topindex_chain {
|
||||
// int level; /* sort level */
|
||||
// char *category; /* category */
|
||||
// char name[2048]; /* path */
|
||||
// struct topindex_chain *next; /* next element */
|
||||
//};
|
||||
|
||||
/* Tools */
|
||||
|
||||
@@ -293,7 +283,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2];
|
||||
char *curr;
|
||||
|
||||
//int n=0;
|
||||
char *a;
|
||||
int slash = 0;
|
||||
|
||||
@@ -338,7 +327,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
if (*curr == '/')
|
||||
curr++;
|
||||
l = link;
|
||||
//c=curr;
|
||||
// couper ce qui est commun
|
||||
while((streql(*link, *curr)) && (*link != 0)) {
|
||||
link++;
|
||||
@@ -350,8 +338,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
link--;
|
||||
curr--;
|
||||
}
|
||||
//if (*link=='/') link++;
|
||||
//if (*curr=='/') curr++;
|
||||
}
|
||||
|
||||
// calculer la profondeur du répertoire courant et remonter
|
||||
@@ -362,7 +348,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
while(*a)
|
||||
if (*(a++) == '/')
|
||||
strlcatbuff(s, "../", ssize);
|
||||
//if (strlen(s)==0) strcatbuff(s,"/");
|
||||
|
||||
if (slash)
|
||||
strlcatbuff(s, "/", ssize); // keep it absolute!
|
||||
|
||||
@@ -639,8 +639,7 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
|
||||
for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien
|
||||
int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel)
|
||||
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
// int ok=0; // OPTI
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
ok = 0;
|
||||
switch (j) {
|
||||
case 0: // prioritaire
|
||||
|
||||
@@ -188,12 +188,10 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
/* Doit-on traiter les non html? */
|
||||
if ((opt->getmode & HTS_GETMODE_NONHTML) == 0) { // non on ne doit pas
|
||||
if (!ishtml(opt, fil)) { // non il ne faut pas
|
||||
//adr[0]='\0'; // ne pas traiter ce lien, pas traiter
|
||||
if (!ishtml(opt, fil)) { // non il ne faut pas
|
||||
forbidden_url = 1; // interdire récupération du lien
|
||||
hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr,
|
||||
fil);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,7 +350,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
} else { // adresse différente, sortir?
|
||||
|
||||
//if (!opt->wizard) { // mode non wizard
|
||||
// doit-on traiter ce lien?.. vérifier droits de sortie
|
||||
switch ((opt->travel & HTS_TRAVEL_SCOPE_MASK)) {
|
||||
case HTS_TRAVEL_SAME_ADDRESS:
|
||||
@@ -380,7 +377,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
if ((i > 0) && (j > 0)) {
|
||||
if (!strfield2(adr + i, urladr() + j)) { // !=
|
||||
if (!opt->wizard) { // mode non wizard
|
||||
// printf("refused: %s\n",adr);
|
||||
forbidden_url = 1; // pas même domaine
|
||||
hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s",
|
||||
adr, fil);
|
||||
@@ -408,7 +404,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
if ((i > 0) && (j > 0)) {
|
||||
if (!strfield2(adr + i, urladr() + j)) { // !-
|
||||
if (!opt->wizard) { // mode non wizard
|
||||
// printf("refused: %s\n",adr);
|
||||
forbidden_url = 1; // pas même .xx
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"foreign location link canceled: %s%s", adr, fil);
|
||||
@@ -439,7 +434,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
// récupérer les liens à côtés d'un lien (nearlink) (nvelle pos)
|
||||
if (forbidden_url != 0 && opt->nearlink) {
|
||||
if (!ishtml(opt, fil)) { // non html
|
||||
//printf("ok %s%s\n",ad,fil);
|
||||
forbidden_url = 0; // autoriser
|
||||
may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner
|
||||
hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil);
|
||||
@@ -887,7 +881,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
}
|
||||
}
|
||||
}
|
||||
//adr[0]='\0'; // cancel
|
||||
}
|
||||
// -------------------- FINAL PHASE --------------------
|
||||
// Test if the "Near" test won
|
||||
|
||||
@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
/* ZLib */
|
||||
#include "zlib.h"
|
||||
//#include "zutil.h"
|
||||
|
||||
/* MiniZip */
|
||||
#include "minizip/zip.h"
|
||||
|
||||
@@ -54,7 +54,6 @@ static int linput(FILE * fp, char *s, int max);
|
||||
#include "htswrap.h"
|
||||
|
||||
/* specific definitions */
|
||||
//#include "htsbase.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -454,8 +453,7 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b
|
||||
for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien
|
||||
int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel)
|
||||
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
// int ok=0; // OPTI
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
ok = 0;
|
||||
switch (j) {
|
||||
case 0: // prioritaire
|
||||
@@ -773,28 +771,7 @@ static void sig_finish(int code) { // finir et quitter
|
||||
fprintf(stderr, "\nExit requested to engine (signal %d)\n", code);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
#if 0
|
||||
static void sig_ask(int code) { // demander
|
||||
char s[256];
|
||||
|
||||
signal(code, sig_term); // quitter si encore
|
||||
printf("\nQuit program/Interrupt/Cancel? (Q/I/C) ");
|
||||
fflush(stdout);
|
||||
scanf("%s", s);
|
||||
if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
|
||||
|| (s[0] == 'q') || (s[0] == 'Q'))
|
||||
exit(0); // quitter
|
||||
else if ((s[0] == 'i') || (s[0] == 'I')) {
|
||||
if (global_opt != NULL) {
|
||||
// ask for stop
|
||||
global_opt->state.stop = 1;
|
||||
}
|
||||
}
|
||||
signal(code, sig_ask); // remettre signal
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
#ifndef _WIN32
|
||||
static void sig_doback(int blind);
|
||||
static void sig_back(int code) { // ignorer et mettre en backing
|
||||
if (global_opt != NULL && !global_opt->background_on_suspend) {
|
||||
@@ -809,36 +786,6 @@ static void sig_back(int code) { // ignorer et mettre en backing
|
||||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
static void sig_ask(int code) { // demander
|
||||
char s[256];
|
||||
|
||||
signal(code, sig_term); // quitter si encore
|
||||
printf
|
||||
("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) ");
|
||||
fflush(stdout);
|
||||
scanf("%s", s);
|
||||
if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
|
||||
|| (s[0] == 'q') || (s[0] == 'Q'))
|
||||
exit(0); // quitter
|
||||
else if ((s[0] == 'b') || (s[0] == 'B') || (s[0] == 'a') || (s[0] == 'A'))
|
||||
sig_doback(0); // arrière plan
|
||||
else if ((s[0] == 'l') || (s[0] == 'L'))
|
||||
sig_doback(1); // arrière plan
|
||||
else if ((s[0] == 'i') || (s[0] == 'I')) {
|
||||
if (global_opt != NULL) {
|
||||
// ask for stop
|
||||
printf("finishing pending transfers.. please wait\n");
|
||||
global_opt->state.stop = 1;
|
||||
}
|
||||
signal(code, sig_ask); // remettre signal
|
||||
} else {
|
||||
printf("cancel..\n");
|
||||
signal(code, sig_ask); // remettre signal
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
static void sig_brpipe(int code) { // treat if necessary
|
||||
signal(code, sig_brpipe);
|
||||
}
|
||||
@@ -951,23 +898,12 @@ static void sig_leave(int code) {
|
||||
|
||||
static void signal_handlers(void) {
|
||||
#ifdef _WIN32
|
||||
#if 0 /* BUG366763 */
|
||||
signal(SIGINT, sig_ask); // ^C
|
||||
#else
|
||||
signal(SIGINT, sig_leave); // ^C
|
||||
#endif
|
||||
signal(SIGTERM, sig_finish); // kill <process>
|
||||
#else
|
||||
#if 0 /* BUG366763 */
|
||||
signal(SIGHUP, sig_back); // close window
|
||||
#endif
|
||||
signal(SIGTSTP, sig_back); // ^Z
|
||||
signal(SIGTERM, sig_finish); // kill <process>
|
||||
#if 0 /* BUG366763 */
|
||||
signal(SIGINT, sig_ask); // ^C
|
||||
#else
|
||||
signal(SIGINT, sig_leave); // ^C
|
||||
#endif
|
||||
signal(SIGPIPE, sig_brpipe); // broken pipe (write into non-opened socket)
|
||||
signal(SIGCHLD, sig_ignore); // child change status
|
||||
#endif
|
||||
|
||||
@@ -250,13 +250,6 @@ static int gethost(const char *hostname, SOCaddr * server) {
|
||||
struct addrinfo hints;
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
#if 0
|
||||
if (IPV6_resolver == 1) // V4 only (for bogus V6 entries)
|
||||
hints.ai_family = PF_INET;
|
||||
else if (IPV6_resolver == 2) // V6 only (for testing V6 only)
|
||||
hints.ai_family = PF_INET6;
|
||||
else
|
||||
#endif
|
||||
hints.ai_family = PF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
hints.ai_protocol = IPPROTO_TCP;
|
||||
@@ -340,12 +333,7 @@ int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort,
|
||||
T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM);
|
||||
|
||||
if (soc != INVALID_SOCKET && socICP != INVALID_SOCKET) {
|
||||
//char url[HTS_URLMAXSIZE * 2];
|
||||
//char method[32];
|
||||
//char data[32768];
|
||||
|
||||
//url[0] = method[0] = data[0] = '\0';
|
||||
//
|
||||
printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort);
|
||||
printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort);
|
||||
#ifndef _WIN32
|
||||
@@ -950,7 +938,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
char *command;
|
||||
char *proto;
|
||||
char *surl;
|
||||
//int directHit = 0;
|
||||
int headRequest = 0;
|
||||
int listRequest = 0;
|
||||
|
||||
@@ -1092,10 +1079,8 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
|
||||
/* Post-process request */
|
||||
if (link_has_authority(surl)) {
|
||||
if (strncasecmp
|
||||
(surl, "http://proxytrack/",
|
||||
sizeof("http://proxytrack/") - 1) == 0) {
|
||||
//directHit = 1; /* Another direct hit hack */
|
||||
if (strncasecmp(surl, "http://proxytrack/",
|
||||
sizeof("http://proxytrack/") - 1) == 0) {
|
||||
}
|
||||
StringCopy(url, surl);
|
||||
} else {
|
||||
@@ -1116,7 +1101,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
toHit += 7;
|
||||
}
|
||||
/* Direct hit */
|
||||
//directHit = 1;
|
||||
StringCopy(url, "");
|
||||
if (!link_has_authority(toHit))
|
||||
StringCat(url, "http://");
|
||||
@@ -1127,7 +1111,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
const char *toHit = surl + sizeof("/proxytrack/") - 1;
|
||||
|
||||
/* Direct hit */
|
||||
//directHit = 1;
|
||||
StringCopy(url, "");
|
||||
if (!link_has_authority(toHit))
|
||||
StringCat(url, "http://");
|
||||
@@ -1611,10 +1594,7 @@ static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) {
|
||||
unsigned char Opcode = buffer[0];
|
||||
unsigned char Version = buffer[1];
|
||||
unsigned short Message_Length = READ_NET16(&buffer[2]);
|
||||
unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
|
||||
//unsigned int Options = READ_NET32(&buffer[8]);
|
||||
//unsigned int Option_Data = READ_NET32(&buffer[12]); /* ICP_FLAG_SRC_RTT */
|
||||
//unsigned int Sender_Host_Address = READ_NET32(&buffer[16]); /* ignored */
|
||||
unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
|
||||
unsigned char *Payload = &buffer[20];
|
||||
|
||||
buffer[bufferSize] = '\0'; /* Ensure payload is NULL terminated */
|
||||
|
||||
@@ -1035,7 +1035,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
if (unzOpenCurrentFile(index->zFile) == Z_OK) {
|
||||
char headerBuff[8192 + 2];
|
||||
int readSizeHeader;
|
||||
//int totalHeader = 0;
|
||||
int dataincache = 0;
|
||||
|
||||
/* For BIG comments */
|
||||
@@ -1074,9 +1073,8 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified
|
||||
ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag
|
||||
ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition",
|
||||
r->cdispo); // Content-disposition
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename
|
||||
if (line[0] != '\0') {
|
||||
int len = r->headers ? ((int) strlen(r->headers)) : 0;
|
||||
@@ -1090,8 +1088,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
strcat(r->headers, "\r\n");
|
||||
}
|
||||
}
|
||||
} while(offset < readSizeHeader && !lineEof);
|
||||
//totalHeader = offset;
|
||||
} while (offset < readSizeHeader && !lineEof);
|
||||
|
||||
/* Previous entry */
|
||||
if (previous_save_[0] != '\0') {
|
||||
@@ -1202,7 +1199,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
strcpy(r->msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
*(r->adr + r->size) = '\0';
|
||||
//printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode);
|
||||
} else { // erreur
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Memory Error");
|
||||
@@ -1484,14 +1480,12 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
a += cache_brstr(a, firstline);
|
||||
strcpy(cache->lastmodified, firstline);
|
||||
} else {
|
||||
// fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version);
|
||||
fclose(cache->dat);
|
||||
cache->dat = NULL;
|
||||
free(use);
|
||||
use = NULL;
|
||||
}
|
||||
} else { // non supporté
|
||||
// fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline);
|
||||
} else { // non supporté
|
||||
fclose(cache->dat);
|
||||
cache->dat = NULL;
|
||||
free(use);
|
||||
@@ -1500,7 +1494,6 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
/* */
|
||||
} else { // Vieille version du cache
|
||||
/* */
|
||||
// hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format");
|
||||
cache->version = 0; // cache 1.0
|
||||
strcpy(cache->lastmodified, firstline);
|
||||
}
|
||||
|
||||
@@ -106,8 +106,6 @@ grep -Eq 'srcset="j\.gif 2x"' "$saved" ||
|
||||
# inline style attribute, with the URL unquoted, double-quoted and single-quoted
|
||||
# (the quote style is preserved on rewrite). No-detect attributes (title, alt,
|
||||
# ...) are left untouched. Asserted by rewrite (deterministic), not download.
|
||||
# data-* (#201/#203) is omitted: its detection is currently nondeterministic and
|
||||
# can't be locked yet.
|
||||
site2="$tmp/attrs"
|
||||
mkdir -p "$site2"
|
||||
for f in xl ibg ibgs cex cexd cexs tt; do gif "$site2/$f.gif"; done
|
||||
@@ -352,4 +350,64 @@ found "v.webm" "$out10"
|
||||
found "subs.vtt" "$out10"
|
||||
notfound "plain.gif" "$out10"
|
||||
|
||||
# Unknown-attr (data-*) URLs (#201/#203): the script automaton state must reset at
|
||||
# </script>, or detection dies for the rest of the page after the first script.
|
||||
site11="$tmp/dataattr"
|
||||
mkdir -p "$site11"
|
||||
for f in pre post mid spdata handler handler2 jsdecoy jsdecoy2 nodecoy \
|
||||
spalt glalt spxml jtail1 jtail2 textdecoy tagdecoy phantom; do gif "$site11/$f.gif"; done
|
||||
cat >"$site11/index.html" <<EOF
|
||||
<html><body>
|
||||
<img data-pre="file://$site11/pre.gif">
|
||||
<script>var x = 1;</script>
|
||||
<img data-post="file://$site11/post.gif">
|
||||
<img data-mid="file://$site11/mid.gif" alt="mid-tag attr, no delimiter after">
|
||||
<img data-sp = "file://$site11/spdata.gif" q=1>
|
||||
<img alt="nodecoy.gif" src="pre.gif">
|
||||
<img alt = "spalt.gif" id=x>
|
||||
<img src="pre.gif"alt="glalt.gif" id=x>
|
||||
<p xmlns:bar = "spxml.gif" id=x></p>
|
||||
<div data-json='["jtail1.gif","jtail2.gif"]' q=1></div>
|
||||
<p>t = "textdecoy.gif" q</p>
|
||||
<img ="tagdecoy.gif" q>
|
||||
<a onclick='q = 1; "h1'>x</a>
|
||||
<img data-h1="file://$site11/handler.gif">
|
||||
<a onclick='w = 1; "h2>x</a>
|
||||
<img data-h2="file://$site11/handler2.gif">
|
||||
<script>var s; s = 1; "jsdecoy.gif";</script>
|
||||
<script>var s2 = "jsdecoy2.gif" x;</script>
|
||||
<script>var f = "</script>
|
||||
<img alt="y" "phantom.gif">
|
||||
</body></html>
|
||||
EOF
|
||||
out11="$tmp/dataattr-out"
|
||||
crawl "$site11/index.html" "$out11"
|
||||
saved11=$(savedhtml "$out11")
|
||||
test -n "$saved11" || ! echo "FAIL: saved dataattr page not found" || exit 1
|
||||
grep -Fq 'data-pre="pre.gif"' "$saved11" ||
|
||||
! echo "FAIL #201: data-* URL before any script not detected/rewritten" || exit 1
|
||||
grep -Fq 'data-post="post.gif"' "$saved11" ||
|
||||
! echo "FAIL #201: data-* URL after a script not detected (state leak)" || exit 1
|
||||
grep -Fq 'data-mid="mid.gif"' "$saved11" ||
|
||||
! echo "FAIL #201: mid-tag data-* URL not detected" || exit 1
|
||||
grep -Fq 'data-sp = "spdata.gif"' "$saved11" ||
|
||||
! echo "FAIL #201: spaced-= data-* URL not detected" || exit 1
|
||||
found "handler.gif" "$out11" # automaton reset at the handler-terminator exit
|
||||
found "handler2.gif" "$out11" # ... and at the '>' exit of an unterminated handler
|
||||
# a JS string not preceded by =/(/, is still ignored after the reset
|
||||
notfound "jsdecoy.gif" "$out11"
|
||||
# in-script, the strict follower gate still applies (intag is 1 in script bodies)
|
||||
notfound "jsdecoy2.gif" "$out11"
|
||||
# no-detect attrs stay exempt, incl. spaced-'=' and glued-attr forms
|
||||
notfound "nodecoy.gif" "$out11"
|
||||
notfound "spalt.gif" "$out11"
|
||||
notfound "glalt.gif" "$out11"
|
||||
notfound "spxml.gif" "$out11"
|
||||
# not '='-preceded: comma-list tails and out-of-tag text tokens stay ignored
|
||||
notfound "jtail2.gif" "$out11"
|
||||
notfound "textdecoy.gif" "$out11"
|
||||
notfound "tagdecoy.gif" "$out11" # a '=' glued to the tag name is not an attr
|
||||
# a </script> inside a JS string must not cause phantom fetches later
|
||||
notfound "phantom.gif" "$out11"
|
||||
|
||||
exit 0
|
||||
|
||||
17
tests/01_engine-reconcile.test
Normal file
17
tests/01_engine-reconcile.test
Normal file
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Cache generation reconcile policies (httrack -#test=reconcile <dir>):
|
||||
# promote a stranded old generation, keep the larger one after an aborted
|
||||
# run, and restore the old one when an update transferred nothing.
|
||||
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
out=$(httrack -#test=reconcile "$dir")
|
||||
|
||||
test "$out" = "cache-reconcile: OK" || {
|
||||
echo "expected 'cache-reconcile: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
19
tests/01_zlib-cache-corrupt.test
Normal file
19
tests/01_zlib-cache-corrupt.test
Normal file
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Read-side cache corruption (httrack -#test=cache-corrupt <dir>): zip byte
|
||||
# surgery (bad/oversized X-Size, blanked X-In-Cache, smashed header, garbled
|
||||
# deflate) must each be rejected per-entry, never crash, never taint the sibling.
|
||||
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# the smashed-header case logs expected "Corrupted cache entry" warnings on
|
||||
# stdout; the verdict is the last line
|
||||
out=$(httrack -#test=cache-corrupt "$dir" 2>/dev/null | tail -n1)
|
||||
|
||||
test "$out" = "cache-corrupt: OK" || {
|
||||
echo "expected 'cache-corrupt: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -4,10 +4,9 @@
|
||||
# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
|
||||
# tool flags despite the #!/bin/bash above.
|
||||
|
||||
# Cache write-failure handling (httrack -#test=cache-writefail <dir>). #174/#219.
|
||||
# A failing new.zip write (disk full) used to crash the process via assertf; it
|
||||
# must instead stop the mirror with a fatal error (exit_xh=-1), no crash. The
|
||||
# self-test asserts that; reverting the fix makes -#test=cache-writefail abort (SIGABRT) and fail.
|
||||
# Cache write-failure policy (-#test=cache-writefail <dir>). #174/#219: disk
|
||||
# full or a failure streak aborts cleanly; an isolated failure or an oversized
|
||||
# entry is only dropped.
|
||||
|
||||
set -eu
|
||||
|
||||
@@ -22,3 +21,9 @@ printf '%s\n' "$out" | grep -qx "cache-writefail: OK" || {
|
||||
echo "expected 'cache-writefail: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# A skipped entry must be warned about with its URL.
|
||||
printf '%s\n' "$out" | grep -q "entry not cached: example.com/" || {
|
||||
echo "expected a URL-bearing skip warning" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Issues #32/#41: a Content-Length that disagrees with the body warns "bogus
|
||||
# state (broken size)" and skips the cache; -%B (tolerant) accepts it.
|
||||
# Issues #32/#41: a Content-Length that disagrees with the body warns
|
||||
# "incomplete transfer" and skips the cache; -%B (tolerant) accepts it.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# Default: warn, but the file is still written.
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'size/oversize.bin' \
|
||||
--log-found 'bogus state \(broken size' \
|
||||
--log-found 'incomplete transfer \(expected' \
|
||||
httrack 'BASEURL/size/index.html'
|
||||
|
||||
# -%B (tolerant): no warning, file written.
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
--found 'size/oversize.bin' \
|
||||
--log-not-found 'bogus state' \
|
||||
--log-not-found 'incomplete transfer|not cached' \
|
||||
httrack 'BASEURL/size/index.html' '-%B'
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# Degenerate delayed-type paths (#5/#107 family): redirects that never resolve
|
||||
# a name must drop cleanly -- no .delayed leftovers (audited by local-crawl.sh),
|
||||
# no "bogus state" cache warnings, resolvable links still land correctly.
|
||||
# no "not cached" warnings, resolvable links still land correctly.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -16,5 +16,5 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun --errors 0 \
|
||||
--not-found 'delayed/noloc.html' \
|
||||
--not-found 'delayed/selfloop.html' \
|
||||
--not-found 'delayed/chain9.pdf' \
|
||||
--log-not-found 'bogus state' \
|
||||
--log-not-found 'not cached' \
|
||||
httrack 'BASEURL/delayed/index.html'
|
||||
|
||||
@@ -7,10 +7,8 @@ set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
|
||||
start=$(date +%s)
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--skip-delayed-audit \
|
||||
--log-found 'More than 2 seconds passed' \
|
||||
httrack 'BASEURL/trickle/index.html' -E2 -c4
|
||||
wall=$(($(date +%s) - start))
|
||||
|
||||
@@ -44,7 +44,7 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
|
||||
--file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
|
||||
--file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \
|
||||
--file-not-matches 'big/p/1.html' 'href="/big/' \
|
||||
--log-not-found 'bogus state|[Pp]anic|assert' \
|
||||
--log-not-found 'not cached|[Pp]anic|assert' \
|
||||
--log-found '\(404\) at link [^ ]*/big/e/404\.html' \
|
||||
--log-found '\(410\) at link [^ ]*/big/e/410\.html' \
|
||||
--log-found '\(500\) at link [^ ]*/big/e/500\.html' \
|
||||
|
||||
12
tests/37_local-cache-outage.test
Normal file
12
tests/37_local-cache-outage.test
Normal file
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An update run against a dead server must not destroy the cache: the no-data
|
||||
# rollback restores the previous hts-cache generation (zip caches lost it).
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun-dead \
|
||||
--found 'simple/basic.html' \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
14
tests/38_local-update-304.test
Normal file
14
tests/38_local-update-304.test
Normal file
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An all-304 update of a tiny site (headers under the 32K rollback threshold)
|
||||
# is a healthy run: it must not trip the no-data rollback as a fake outage.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
|
||||
--log-found 'no files updated' \
|
||||
--log-not-found 'No data seems to have been transferred' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html'
|
||||
13
tests/39_local-delayed-cancel.test
Normal file
13
tests/39_local-delayed-cancel.test
Normal file
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Cancelled delayed-type-checks must not orphan .delayed placeholders (#483).
|
||||
# Timing-dependent (hence two tries); -A keeps the window reachable.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
for _ in 1 2; do
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
httrack 'BASEURL/dcancel/index.html' -E1 -c4 -A25000
|
||||
done
|
||||
@@ -48,6 +48,7 @@ TESTS = \
|
||||
01_engine-parse.test \
|
||||
01_engine-pause.test \
|
||||
01_engine-rcfile.test \
|
||||
01_engine-reconcile.test \
|
||||
01_engine-redirect.test \
|
||||
01_engine-relative.test \
|
||||
01_engine-robots.test \
|
||||
@@ -63,6 +64,7 @@ TESTS = \
|
||||
01_engine-useragent.test \
|
||||
01_zlib-acceptencoding.test \
|
||||
01_zlib-cache.test \
|
||||
01_zlib-cache-corrupt.test \
|
||||
01_zlib-cache-golden.test \
|
||||
01_zlib-cache-writefail.test \
|
||||
01_zlib-savename-cached.test \
|
||||
@@ -99,6 +101,9 @@ TESTS = \
|
||||
33_local-delayed.test \
|
||||
34_local-maxtime.test \
|
||||
35_local-maxsize.test \
|
||||
36_local-bigcrawl.test
|
||||
36_local-bigcrawl.test \
|
||||
37_local-cache-outage.test \
|
||||
38_local-update-304.test \
|
||||
39_local-delayed-cancel.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -25,6 +25,8 @@
|
||||
# --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
|
||||
# which the ephemeral port forces into the cookie domain) and passes it to
|
||||
# httrack via --cookies-file, to exercise preloaded cookies.
|
||||
# --rerun-dead re-runs with the server stopped: the no-data rollback must
|
||||
# restore the previous hts-cache generation byte-identical.
|
||||
|
||||
set -u
|
||||
|
||||
@@ -37,6 +39,7 @@ key="${testdir}/server.key"
|
||||
tls=
|
||||
verbose=
|
||||
rerun=
|
||||
rerun_dead=
|
||||
tmpdir=
|
||||
serverpid=
|
||||
crawlpid=
|
||||
@@ -94,7 +97,6 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
|
||||
# --- parse leading control flags --------------------------------------------
|
||||
declare -a audit=()
|
||||
declare -a cookies=()
|
||||
skip_delayed_audit=""
|
||||
scheme=http
|
||||
pos=0
|
||||
args=("$@")
|
||||
@@ -102,7 +104,8 @@ nargs=$#
|
||||
while test "$pos" -lt "$nargs"; do
|
||||
case "${args[$pos]}" in
|
||||
--debug) verbose=1 ;;
|
||||
--rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
|
||||
--rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
|
||||
--rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
|
||||
--no-purge)
|
||||
nopurge=1
|
||||
audit+=("--no-purge")
|
||||
@@ -119,9 +122,6 @@ while test "$pos" -lt "$nargs"; do
|
||||
pos=$((pos + 1))
|
||||
cookies+=("${args[$pos]}")
|
||||
;;
|
||||
--skip-delayed-audit)
|
||||
skip_delayed_audit=1
|
||||
;;
|
||||
--errors | --files)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
@@ -241,6 +241,43 @@ if test -n "$rerun"; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- optional dead pass: server stopped, the cache must survive the rollback --
|
||||
if test -n "$rerun_dead"; then
|
||||
zip="${out}/hts-cache/new.zip"
|
||||
test -s "$zip" || die "no cache was written by the first pass"
|
||||
cp "$zip" "${tmpdir}/cache-before.zip"
|
||||
cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
serverpid=
|
||||
info "re-running httrack against the stopped server"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid" || true
|
||||
crawlpid=
|
||||
result "OK (dead pass ran)"
|
||||
# The dead pass must have gone through the no-data rollback, not bailed out
|
||||
# before the mirror loop (which would leave the cache trivially untouched).
|
||||
info "checking the dead pass hit the rollback"
|
||||
if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
|
||||
result "OK"
|
||||
else
|
||||
result "rollback notice not found in hts-log.txt"
|
||||
exit 1
|
||||
fi
|
||||
info "checking the previous cache generation was restored"
|
||||
if cmp -s "$zip" "${tmpdir}/cache-before.zip" &&
|
||||
test ! -e "${out}/hts-cache/old.zip"; then
|
||||
result "OK"
|
||||
else
|
||||
result "new.zip differs from the pre-outage cache (or old.zip left behind)"
|
||||
exit 1
|
||||
fi
|
||||
# Audits below describe the healthy crawl, not the dead pass.
|
||||
cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
|
||||
fi
|
||||
|
||||
# --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
|
||||
hostroot=
|
||||
for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
|
||||
@@ -252,15 +289,12 @@ done
|
||||
test -n "$hostroot" || die "could not find host root under $out"
|
||||
debug "host root: $hostroot"
|
||||
|
||||
# A completed crawl must leave no .delayed temporaries (issue #107).
|
||||
# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
|
||||
if test -z "$skip_delayed_audit"; then
|
||||
info "checking for leftover .delayed files"
|
||||
leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
|
||||
if test -z "$leftovers"; then result "OK"; else
|
||||
result "leftover: $leftovers"
|
||||
exit 1
|
||||
fi
|
||||
# No crawl, even a cancelled one, may leave .delayed temporaries (#107, #483).
|
||||
info "checking for leftover .delayed files"
|
||||
leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
|
||||
if test -z "$leftovers"; then result "OK"; else
|
||||
result "leftover: $leftovers"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- audit -------------------------------------------------------------------
|
||||
|
||||
@@ -781,7 +781,7 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
self.send_raw(b"", "text/html")
|
||||
|
||||
# broken Content-Length (#32/#41): declared size != bytes sent. httrack
|
||||
# warns "bogus state (broken size)" and skips the cache unless -%B.
|
||||
# warns "incomplete transfer" and skips the cache unless -%B.
|
||||
def route_size_index(self):
|
||||
self.send_html('\t<a href="oversize.bin">over</a>\n')
|
||||
|
||||
@@ -831,6 +831,16 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
def route_redir_target(self):
|
||||
self.send_raw(b"<html><body>redirect target</body></html>\n", "text/html")
|
||||
|
||||
# --- /mini304/: tiny fully-cacheable site (an update gets only 304s) ---
|
||||
def route_mini304_index(self):
|
||||
self.big_send(
|
||||
b'<html><body>\n\t<a href="page.html">page</a>\n</body></html>\n',
|
||||
"text/html",
|
||||
)
|
||||
|
||||
def route_mini304_page(self):
|
||||
self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
|
||||
|
||||
# --- delayed-type degenerate paths (issues #5/#107) --------------------
|
||||
def route_delayed_index(self):
|
||||
self.send_html(
|
||||
@@ -903,6 +913,26 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# #483: trickled .bin pages so the -E stop lands in the type waiter's
|
||||
# unlock-to-patch window with body bytes pending.
|
||||
def route_dcancel_index(self):
|
||||
self.send_bin_index()
|
||||
|
||||
def route_dcancel_page(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", "4096")
|
||||
self.end_headers()
|
||||
if self.command == "HEAD":
|
||||
return
|
||||
try:
|
||||
for _ in range(32):
|
||||
self.wfile.write(b"z" * 128)
|
||||
self.wfile.flush()
|
||||
time.sleep(0.05)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# -M byte cap (#77): large fast files so a crawl overruns -M immediately.
|
||||
BIGFILE_BYTES = 640 * 1024
|
||||
|
||||
@@ -966,6 +996,15 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/trickle/p5.bin": route_trickle_page,
|
||||
"/trickle/p6.bin": route_trickle_page,
|
||||
"/trickle/p7.bin": route_trickle_page,
|
||||
"/dcancel/index.html": route_dcancel_index,
|
||||
"/dcancel/p0.bin": route_dcancel_page,
|
||||
"/dcancel/p1.bin": route_dcancel_page,
|
||||
"/dcancel/p2.bin": route_dcancel_page,
|
||||
"/dcancel/p3.bin": route_dcancel_page,
|
||||
"/dcancel/p4.bin": route_dcancel_page,
|
||||
"/dcancel/p5.bin": route_dcancel_page,
|
||||
"/dcancel/p6.bin": route_dcancel_page,
|
||||
"/dcancel/p7.bin": route_dcancel_page,
|
||||
"/bigfiles/index.html": route_bigfiles_index,
|
||||
"/bigfiles/p0.bin": route_bigfile,
|
||||
"/bigfiles/p1.bin": route_bigfile,
|
||||
@@ -993,6 +1032,8 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/redir/index.html": route_redir_index,
|
||||
"/redir/go.php": route_redir_go,
|
||||
"/redir/target.html": route_redir_target,
|
||||
"/mini304/index.html": route_mini304_index,
|
||||
"/mini304/page.html": route_mini304_page,
|
||||
}
|
||||
|
||||
# --- /big/ seeded pseudo-site ------------------------------------------
|
||||
|
||||
@@ -236,12 +236,15 @@ main() {
|
||||
|
||||
# The real lintian gate (debuild only reports, it does not fail on tags).
|
||||
# --profile debian: CI runners are Ubuntu, whose vendor data would wrongly
|
||||
# reject the Debian "unstable" distribution. newer-standards-version only
|
||||
# means the local lintian is older than the buildds', not a package
|
||||
# defect, so suppress it. set -e turns any error/warning tag into a failure.
|
||||
# reject the Debian "unstable" distribution. Suppressed tags are stale-local-
|
||||
# lintian skew, not package defects: newer-standards-version, and
|
||||
# recommended-field (old lintian still wants the Priority field the sid
|
||||
# lintian in CI accepts dropping). set -e turns any error/warning tag into
|
||||
# a failure.
|
||||
info "running lintian gate (--fail-on=error,warning)"
|
||||
lintian --profile debian -I -i --fail-on=error,warning \
|
||||
--suppress-tags newer-standards-version "${changes[@]}"
|
||||
--suppress-tags newer-standards-version,recommended-field \
|
||||
"${changes[@]}"
|
||||
|
||||
dcmd cp -- "${changes[@]}" "$outdir/"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user