Compare commits

...

2 Commits

Author SHA1 Message Date
Xavier Roche
cf483a24e4 Remove commented-out code across the engine
Sweep of ~650 lines of //-disabled statements, declarations, and calls
(plus three /*-wrapped dead blocks in htscoremain.c and htscore.c and
the MFC pApp remnant in htsserver.c). Prose comments stay, including
labels that describe live code; usage examples and config-knob docs
(HTS_TRACE_MALLOC, the htsweb.c DEBUG toggle) were kept. Insertions are
clang-format rewraps of live lines that became touched by the deletions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-06 10:42:28 +02:00
Xavier Roche
abaea566bd Remove dead #if 0 blocks and their orphaned satellites
All #if 0 code that has been disabled for years goes away: the sig_ask
interactive signal handlers, check_rate, HTS_TOTAL_RECV_CHECK, the
make_empty_index no-op body (function removed with its only call site),
the htscharset sample main(), the dead FTP CWD sequence, and assorted
disabled fragments in the parser and engine. Blocks with a live #else
keep that branch. The htsweb.c DEBUG toggle stays: its #else arm is the
live definition and the macro is used.

The htsparse.c 3.43 note about not post-excluding authorized links is
kept as a plain comment; commented-out declarations and calls that only
served the removed blocks go with them. French comments rewrapped by
the formatter were translated in passing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-06 10:16:21 +02:00
32 changed files with 258 additions and 1447 deletions

View File

@@ -283,7 +283,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
char *position;
int need_param = 1;
//int return_param=0;
int pos;
command[0] = param[0] = addcommand[0] = '\0';
@@ -361,7 +360,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
strlcatbuff(return_argv[0], "0", return_argv_size);
else if (strcmp(param, "on") == 0) {
// on is the default
// strcatbuff(return_argv[0],"1");
} else
strlcatbuff(return_argv[0], param, return_argv_size);
}

View File

@@ -43,14 +43,12 @@ Please visit our Website: http://www.httrack.com
#include "htsback.h"
//#ifdef _WIN32
#include "htsftp.h"
#if HTS_USEZLIB
#include "htszlib.h"
#else
#error HTS_USEZLIB not defined
#endif
//#endif
#ifdef _WIN32
#ifndef __cplusplus
@@ -597,7 +595,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
#if HTS_USEZLIB
if (back[p].r.compressed) {
if (back[p].r.size > 0) {
//if ( (back[p].r.adr) && (back[p].r.size>0) ) {
// stats
back[p].compressed_size = back[p].r.size;
// en mémoire -> passage sur disque
@@ -927,7 +924,6 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
/* clear everything but connection: switch, close, and reswitch */
back_connxfr(src, &tmp);
back_delete(opt, cache, sback, p);
//deletehttp(src);
back_connxfr(&tmp, src);
src->req.flush_garbage = 1; /* ignore CRLF garbage */
return 1;
@@ -1437,7 +1433,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
back[p].url_adr, back[p].url_fil, back[p].url_sav);
}
if (cache != NULL) {
//hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
back_finalize(opt, cache, sback, p);
}
}
@@ -1578,7 +1573,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
if (back[p].r.soc != INVALID_SOCKET) { /* we never know */
deletehttp(&back[p].r);
}
//memset(&(back[p].r), 0, sizeof(htsblk));
hts_init_htsblk(&back[p].r);
back[p].r.location = back[p].location_buffer;
@@ -1586,7 +1580,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
strcpybuff(back[p].url_adr, adr);
strcpybuff(back[p].url_fil, fil);
strcpybuff(back[p].url_sav, save);
//back[p].links_index = links_index;
// copier referer si besoin
strcpybuff(back[p].referer_adr, "");
strcpybuff(back[p].referer_fil, "");
@@ -1825,8 +1818,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
back[p].url_adr, back[p].url_fil);
}
back[p].r.notmodified = 1; // fichier non modifié
back[p].status = STATUS_READY; // OK prêt
//file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); // not modified
back[p].status = STATUS_READY; // OK prêt
back_set_finished(sback, p);
// finalize transfer
@@ -1841,7 +1833,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
} else { // erreur
// effacer r
hts_init_htsblk(&back[p].r);
//memset(&(back[p].r), 0, sizeof(htsblk));
back[p].r.location = back[p].location_buffer;
// et continuer (chercher le fichier)
}
@@ -2036,7 +2027,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
// ouvrir liaison, envoyer requète
// ne pas traiter ou recevoir l'en tête immédiatement
hts_init_htsblk(&back[p].r);
// memset(&(back[p].r), 0, sizeof(htsblk));
back[p].r.location = back[p].location_buffer;
// fresh connect: address list not yet probed, start at the first
sback->connect_fallback[p].addr_index = 0;
@@ -2087,7 +2077,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
#if HTS_USEOPENSSL
else if (strfield(back[p].url_adr, "https://")) { // let's rock
back[p].r.ssl = 1;
// back[p].r.ssl_soc = NULL;
back[p].r.ssl_con = NULL;
}
#endif
@@ -2153,9 +2142,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
back[p].rateout = -1; // pas de gestion (default)
}
// Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
// rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
//if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
// Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
// cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
// page!)
#if CNXDEBUG
printf("Xfopen ok, poll..\n");
@@ -2173,7 +2162,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
if (soc == INVALID_SOCKET) { // erreur socket
back[p].status = STATUS_READY; // FINI
back_set_finished(sback, p);
//if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
back[p].r.soc = INVALID_SOCKET;
} else {
if (!back[p].r.is_file)
@@ -2194,8 +2182,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
// note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
// le lien est considéré comme traité
//if (back[p].soc<0) // erreur
// return -1;
return 0;
} else {
@@ -2268,9 +2254,6 @@ void back_solve(httrackp * opt, lien_back * back) {
} else {
hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
}
//if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
// hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
//}
}
}
@@ -2319,11 +2302,6 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
(void) back_flush_output(opt, cache, sback, i); // flush output buffers
usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
back[i].url_fil);
//if (back[i].links_index >= 0) {
// assertf(back[i].links_index < opt->hash->max_lien);
// opt->hash->liens[back[i].links_index]->pass2 = -1;
// // *back[i].pass2_ptr=-1; // Done!
//}
/* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
{
int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME ); // lecture type 0 (sav)
@@ -2507,9 +2485,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
busy_state = busy_recv = 0;
#if 0
check_rate(stat_timestart, opt->maxrate); // vérifier taux de transfert
#endif
// inscrire les sockets actuelles, et rechercher l'ID la plus élevée
FD_ZERO(&fds);
FD_ZERO(&fds_c);
@@ -2519,8 +2494,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
nfds = INVALID_SOCKET;
max_c = 1;
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
// for(i=0;i<back_max;i++) {
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
unsigned int i = (i_mod + mod_random) % (back_max);
// en cas de gestion du connect préemptif
@@ -2557,8 +2531,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
// poll pour la lecture sur les sockets
if ((back[i].status > 0) && (back[i].status < 100)) { // en réception http
#if BDEBUG==1
//printf("....socket in progress: %d\n",back[i].r.soc);
#if BDEBUG == 1
#endif
// non local et non ftp
if (!back[i].r.is_file) {
@@ -2644,8 +2617,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
busy_recv = 0;
// recevoir les données arrivées
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
// for(i=0;i<back_max;i++) {
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
unsigned int i = (i_mod + mod_random) % (back_max);
if (back[i].status > 0) {
@@ -2785,7 +2757,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
back[i].rateout_time = back[i].ka_time_start;
}
// envoyer header
//if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0) // vrai get
HTS_STAT.stat_nrequests++;
if (!back[i].head_request)
http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2852,7 +2823,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
#if HTS_XGETHOST
else if (back[i].status == STATUS_WAIT_DNS) { // attendre gethostbyname
#if DEBUGDNS
//printf("status 101 for %s\n",back[i].url_adr);
#endif
if (!gestion_timeout)
@@ -3125,8 +3095,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
// Si réception chunk, tester si on est pas à la fin!
if (back[i].status == 1) {
if (back[i].is_chunk) { // attendre prochain chunk
if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
//printf("chunk end at %d\n",back[i].r.size);
if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
back[i].status = STATUS_CHUNK_CR; /* fetch ending CRLF */
if (back[i].chunk_adr != NULL) {
freet(back[i].chunk_adr);
@@ -3189,11 +3158,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
back_finalize(opt, cache, sback, i);
}
if (back[i].r.totalsize >= 0) { // tester totalsize
//if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) { // tester totalsize
if (back[i].r.totalsize >= 0) { // tester totalsize
if (back[i].r.totalsize != back[i].r.size) { // pas la même!
if (!opt->tolerant) {
//#if HTS_CL_IS_FATAL
deleteaddr(&back[i].r);
if (back[i].r.size < back[i].r.totalsize)
back[i].r.statuscode = STATUSCODE_CONNERROR; // recatch
@@ -3202,14 +3169,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
" expected)", (LLint) back[i].r.size,
(LLint) back[i].r.totalsize);
} else {
//#else
// Un warning suffira..
hts_log_print(opt, LOG_WARNING,
"Incorrect length (" LLintP "!=" LLintP
" expected) for %s%s", (LLint) back[i].r.size,
" expected) for %s%s",
(LLint) back[i].r.size,
(LLint) back[i].r.totalsize, back[i].url_adr,
back[i].url_fil);
//#endif
}
}
}
@@ -3336,10 +3302,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
back[i].r.size);
#endif
/* End */
//if (back[i].status==STATUS_CHUNK_CR) {
back[i].status = STATUS_READY; // fin
back[i].status = STATUS_READY; // fin
back_set_finished(sback, i);
//}
// finalize transfer if not temporary
if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3806,7 +3770,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
back[i].url_adr, back[i].url_fil);
// finalize
//file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); // not modified
if (back[i].r.statuscode > 0) {
hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
back_finalize(opt, cache, sback, i);
@@ -3816,37 +3779,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
back[i].url_adr, back[i].url_fil);
#endif
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
} else { // erreur
back[i].status = STATUS_READY; // terminé
back_set_finished(sback, i);
//printf("erreur cache\n");
}
/********** NO - must complete the body! ********** */
#if 0
} else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
|| (back[i].r.statuscode == 412)
|| (back[i].r.statuscode == 416)
) { // Ne pas prendre le html, erreurs connues et gérées
#if HTS_DEBUG_CLOSESOCK
DEBUG_W
("back_wait(301,302,303,307,412,416..): deletehttp\n");
#endif
// Couper connexion
/*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
back_maydeletehttp(opt, cache, sback, i);
back[i].status = STATUS_READY; // terminé
back_set_finished(sback, i);
// finalize
if (back[i].r.statuscode > 0) {
hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
back_finalize(opt, cache, sback, i);
}
#endif
/********** **************************** ********** */
}
// MIME type excluded by a -mime: filter: abort, don't fetch
// the body (#58)
@@ -4020,7 +3957,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
deletehttp(&back[i].r);
}
back[i].r.soc = INVALID_SOCKET;
//back[i].r.statuscode=206; ????????
back[i].r.statuscode = STATUSCODE_NON_FATAL;
if (strnotempty(back[i].r.msg))
strcpybuff(back[i].r.msg,
@@ -4047,8 +3983,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
hts_log_print(opt, LOG_TRACE, "finalizing empty");
back_finalize(opt, cache, sback, i);
}
} else if (!back[i].r.is_chunk) { // pas de chunk
//if (back[i].r.http11!=2) { // pas de chunk
} else if (!back[i].r.is_chunk) { // pas de chunk
back[i].is_chunk = 0;
back[i].status = 1; // start body
} else {
@@ -4109,14 +4044,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
if (back[i].status < 0) {
if (!back[i].testmode) { // pas en test
UNLINK(back[i].url_sav); // éliminer fichier (endommagé)
//printf("&& %s\n",back[i].url_sav);
}
}
#endif
/* funny log for commandline users */
//if (!opt->quiet) {
// petite animation
if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
if (back[i].status == STATUS_READY) {
if (back[i].r.statuscode == HTTP_OK)
@@ -4129,18 +4061,16 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
fflush(stdout);
}
}
//}
} // status>0
} // for
} // for
// vérifier timeouts
if (gestion_timeout) {
TStamp act;
act = time_local(); // temps en secondes
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
// for(i=0;i<back_max;i++) {
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
unsigned int i = (i_mod + mod_random) % (back_max);
if (back[i].status > 0) { // réception/connexion/..
@@ -4168,7 +4098,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
continue;
}
}
//printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
back[i].url_fil);

View File

@@ -55,8 +55,6 @@ Please visit our Website: http://www.httrack.com
// inline function in Wspiapi.h.
#include <ws2tcpip.h>
#include <Wspiapi.h>
// #include <winsock2.h>
// #include <tpipv6.h>
#endif

View File

@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
#if DEBUG_COOK
printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
cook_name, cook_value, domain, path);
//printf(">>>cook: %s<<<\n",cookie->data);
#endif
return 0;
}
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
char catbuff[CATBUFF_SIZE];
char buffer[8192];
// cookie->data[0]='\0';
// Fusionner d'abord les éventuels cookies IE
#ifdef _WIN32
{

View File

@@ -265,8 +265,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
char BIGSTK headers[8192];
int headersSize = 0;
//int entryBodySize = 0;
//int entryFilenameSize = 0;
zip_fileinfo fi;
const char *url_save_suffix = url_save;
int zErr;
@@ -359,8 +357,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename
ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix); // Original save filename
//entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil));
/* Filename */
if (!link_has_authority(url_adr)) {
strcpybuff(filename, "http://");
@@ -725,7 +721,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
char BIGSTK headerBuff[8192 + 2];
int readSizeHeader;
//int totalHeader = 0;
int dataincache = 0;
/* For BIG comments */
@@ -776,13 +771,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
HTS_URLMAXSIZE * 2);
ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo,
sizeof(r.cdispo));
//ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address
//ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
sizeof(previous_save_));
}
} while(offset < readSizeHeader && !lineEof);
//totalHeader = offset;
} while (offset < readSizeHeader && !lineEof);
/* Previous entry */
if (previous_save_[0] != '\0') {
@@ -964,7 +956,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
r.statuscode = STATUSCODE_INVALID;
strcpybuff(r.msg,
"Cache Write Error : Unable to Create File");
//printf("%s\n",save);
}
}
@@ -1042,7 +1033,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
strcpybuff(r.msg, "Cache Read Error : Read Data");
} else
*(r.adr + r.size) = '\0';
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
} else { // erreur
r.statuscode = STATUSCODE_INVALID;
strcpybuff(r.msg, "Cache Memory Error");
@@ -1221,17 +1211,14 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
// sécurité
r.adr = NULL;
r.out = NULL;
////r.location=NULL; non, fixée lors des 301 ou 302
r.fp = NULL;
if ((r.statuscode >= 0) && (r.statuscode <= 999)
&& (r.notmodified >= 0) && (r.notmodified <= 9)) { // petite vérif intégrité
if ((save) && (!header_only)) { /* ne pas lire uniquement header */
//int to_file=0;
if ((save) && (!header_only)) { /* ne pas lire uniquement header */
r.adr = NULL;
r.soc = INVALID_SOCKET;
// // r.location=NULL;
#if HTS_DIRECTDISK
// Court-circuit:
@@ -1240,12 +1227,11 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
int ok = 0;
r.is_write = 1; // écrire
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) { // un fichier existe déja
//if (fsize_utf8(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree)
if (fexist_utf8(fconv(catbuff, sizeof(catbuff),
save))) { // un fichier existe déja
ok = 1; // plus rien à faire
filenote(&opt->state.strc, save, NULL); // noter comme connu
file_notify(opt, adr, fil, save, 0, 0, 0);
//}
}
if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur!
@@ -1296,7 +1282,6 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
r.statuscode = STATUSCODE_INVALID;
strcpybuff(r.msg,
"Cache Write Error : Unable to Create File");
//printf("%s\n",save);
}
}
@@ -1344,14 +1329,13 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
strcpybuff(r.msg, "Cache Read Error : Read Data");
} else
*(r.adr + r.size) = '\0';
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
} else { // erreur
r.statuscode = STATUSCODE_INVALID;
strcpybuff(r.msg, "Cache Memory Error");
}
}
}
} // si save==null, ne rien charger (juste en tête)
} // si save==null, ne rien charger (juste en tête)
} else {
#if DEBUGCA
printf("Cache Read Error : Bad Data");
@@ -1574,25 +1558,10 @@ void cache_init(cache_back * cache, httrackp * opt) {
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
HTS_PROTECT_FOLDER);
#endif
if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) { // il existe déja un cache précédent.. renommer
/* Previous cache from the previous cache version */
#if 0
/* No.. reuse with old httrack releases! */
if (fexist
(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.dat")))
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.dat"));
if (fexist
(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.ndx")))
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.ndx"));
#endif
if ((fexist(fconcat(
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/new.zip")))) { // a previous cache exists.. rename it
/* Previous cache version */
if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
rename(fconcat
@@ -1633,7 +1602,13 @@ void cache_init(cache_back * cache, httrackp * opt) {
} else {
hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
}
} else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
} else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/new.dat"))) &&
(fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/new.ndx")))) { // a previous cache
// exists.. rename it
#if DEBUGCA
printf("work with former cache\n");
#endif
@@ -1662,7 +1637,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
"hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/old.ndx"));
} else { // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel
} else { // one or both cache files missing: remove the remaining one
#if DEBUGCA
printf("new cache\n");
#endif
@@ -2031,12 +2006,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
"hts-cache/new.lst"), "wb");
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
opt->state.strc.lst = cache->lst;
//{
//filecreate_params tmp;
//strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin
//tmp.lst=cache->lst; // fichier lst
//filenote("",&tmp); // initialiser filecreate
//}
// supprimer old.txt
if (fexist
@@ -2125,12 +2094,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
"hts-cache/new.lst"), "wb");
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
opt->state.strc.lst = cache->lst;
//{
// filecreate_params tmp;
// strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin
// tmp.lst=cache->lst; // fichier lst
// filenote("",&tmp); // initialiser filecreate
//}
// supprimer old.txt
if (fexist
@@ -2162,8 +2125,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
"statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"
LF);
}
// test
// cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9);
} // cache->ndx!=NULL
} //cache->zipOutput != NULL

View File

@@ -197,7 +197,6 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
htsblk blkretour;
hts_init_htsblk(&blkretour);
//memset(&blkretour, 0, sizeof(htsblk)); // effacer
blkretour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx
// Lire en têtes restants
sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -207,8 +206,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
}
// CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
//strcatbuff(data,"\r\n");
// CR/LF final de l'en tête inutile car déja placé via la ligne vide
// juste au dessus
if (blkretour.totalsize > 0) {
int len = (int) min(blkretour.totalsize, 32000);
int pos = (int) strlen(data);

View File

@@ -841,71 +841,6 @@ static unsigned int nlz8(unsigned char x) {
} \
} while(0)
/* Sample. */
#if 0
int main(int argc, char **argv) {
int i;
int hex = 0;
#define READ_INT(DEST) \
( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1) \
|| (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
for(i = 1 ; i < argc ; i++) {
unsigned int uc, from, to;
if (strcmp(argv[i], "--hex") == 0) {
hex = 1;
}
else if (strcmp(argv[i], "--decimal") == 0) {
hex = 0;
}
else if (strcmp(argv[i], "--decode") == 0) {
#define RD fgetc_unlocked(stdin)
#define WR(C) do { \
if (C != -1) { \
printf("%04x\n", C); \
} else if (!feof(stdin)) { \
fprintf(stderr, "read error\n"); \
exit(EXIT_FAILURE); \
} \
} while(0)
while(!feof(stdin)) {
READ_UNICODE(RD, WR);
}
#undef RD
#undef WR
}
else if (strcmp(argv[i], "-range") == 0
&& i + 2 < argc
&& (++i, 1)
&& READ_INT(from)
&& (++i, 1)
&& READ_INT(to)
) {
unsigned int i;
for(i = from ; i < to ; i++) {
#define EM(C) fputc_unlocked(C, stdout)
EMIT_UNICODE(i, EM);
#undef EM
}
}
else if (READ_INT(uc)) {
#define EM(C) fputc_unlocked(C, stdout)
EMIT_UNICODE(uc, EM);
#undef EM
}
else {
return EXIT_FAILURE;
}
}
return EXIT_SUCCESS;
}
#endif
/* IDNA helpers. */
#undef ADD_BYTE
#undef INCREASE_CAPA

View File

@@ -54,10 +54,6 @@ Please visit our Website: http://www.httrack.com
// extension par défaut pour fichiers n'en ayant pas
#define DEFAULT_EXT ".html"
#define DEFAULT_EXT_SHORT ".htm"
// #define DEFAULT_BIN_EXT ".bin"
// #define DEFAULT_BIN_EXT_SHORT ".bin"
// #define DEFAULT_EXT ".txt"
// #define DEFAULT_EXT_SHORT ".txt"
// éviter les /nul, /con..
#define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -80,10 +76,6 @@ Please visit our Website: http://www.httrack.com
// always direct-to-disk (0/1)
#define HTS_DIRECTDISK_ALWAYS 1
// gérer une table de hachage?
// REMOVED
// #define HTS_HASH 1
// fast cache (build hash table)
#define HTS_FAST_CACHE 1
@@ -97,27 +89,10 @@ Please visit our Website: http://www.httrack.com
// always transform a '//' into a sigle '/'
#define HTS_STRIP_DOUBLE_SLASH 0
// case-sensitive pour les dossiers et fichiers (0/1)
// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
// pas très utile..
// ..et pas bcp respecté]
// REMOVED
// #define HTS_CASSE 0
// Un fichier ayant une taille différente du content-length doit il être annulé?
// SEE opt.tolerant and opt.http10
// #define HTS_CL_IS_FATAL 0
// une erreur supprime le fichier sur disque
// (non fixé pour cause de retry)
#define HTS_REMOVE_BAD_FILES 0
// en cas de Range: xx- donnant un Content-length: xx
// alors skipper le fichier, considéré comme transmis
// #define HTS_SKIP_FULL_RANGE 1
// nombre max de filtres que l'utilisateur peut fixer
// #define HTS_FILTERSMAX 10000
#define HTS_FILTERSINC 1000
// connect non bloquant? (poll sur write)

View File

@@ -119,46 +119,93 @@ hts_log_print(opt, LOG_INFO, "engine: end"); \
RUN_CALLBACK0(opt, end); \
}
#define XH_extuninit do { \
HTMLCHECK_UNINIT \
hts_record_free(opt); \
if (filters && filters[0]) { \
freet(filters[0]); filters[0]=NULL; \
} \
if (filters) { \
freet(filters); filters=NULL; \
} \
back_delete_all(opt,&cache,sback); \
back_free(&sback); \
checkrobots_free(&robots);\
if (cache.use) { freet(cache.use); cache.use=NULL; } \
if (cache.dat) { fclose(cache.dat); cache.dat=NULL; } \
if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
if (cache.zipOutput) { \
zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
cache.zipOutput = NULL; \
} \
if (cache.zipInput) { \
unzClose(cache.zipInput); \
cache.zipInput = NULL; \
} \
if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
if (opt->log != NULL) fflush(opt->log); \
if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
if (cache_tests) { coucal_delete(&cache_tests); } \
if (template_header) { freet(template_header); template_header=NULL; } \
if (template_body) { freet(template_body); template_body=NULL; } \
if (template_footer) { freet(template_footer); template_footer=NULL; } \
hash_free(&hash); \
clearCallbacks(&opt->state.callbacks); \
/*structcheck_init(-1);*/ \
} while(0)
#define XH_extuninit \
do { \
HTMLCHECK_UNINIT \
hts_record_free(opt); \
if (filters && filters[0]) { \
freet(filters[0]); \
filters[0] = NULL; \
} \
if (filters) { \
freet(filters); \
filters = NULL; \
} \
back_delete_all(opt, &cache, sback); \
back_free(&sback); \
checkrobots_free(&robots); \
if (cache.use) { \
freet(cache.use); \
cache.use = NULL; \
} \
if (cache.dat) { \
fclose(cache.dat); \
cache.dat = NULL; \
} \
if (cache.ndx) { \
fclose(cache.ndx); \
cache.ndx = NULL; \
} \
if (cache.zipOutput) { \
zipClose(cache.zipOutput, \
"Created by HTTrack Website Copier/" HTTRACK_VERSION); \
cache.zipOutput = NULL; \
} \
if (cache.zipInput) { \
unzClose(cache.zipInput); \
cache.zipInput = NULL; \
} \
if (cache.olddat) { \
fclose(cache.olddat); \
cache.olddat = NULL; \
} \
if (cache.lst) { \
fclose(cache.lst); \
cache.lst = opt->state.strc.lst = NULL; \
} \
if (cache.txt) { \
fclose(cache.txt); \
cache.txt = NULL; \
} \
if (opt->log != NULL) \
fflush(opt->log); \
if (makestat_fp) { \
fclose(makestat_fp); \
makestat_fp = NULL; \
} \
if (maketrack_fp) { \
fclose(maketrack_fp); \
maketrack_fp = NULL; \
} \
if (opt->accept_cookie) \
cookie_save(opt->cookie, \
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), \
StringBuff(opt->path_log), "cookies.txt")); \
if (makeindex_fp) { \
fclose(makeindex_fp); \
makeindex_fp = NULL; \
} \
if (cache_hashtable) { \
coucal_delete(&cache_hashtable); \
} \
if (cache_tests) { \
coucal_delete(&cache_tests); \
} \
if (template_header) { \
freet(template_header); \
template_header = NULL; \
} \
if (template_body) { \
freet(template_body); \
template_body = NULL; \
} \
if (template_footer) { \
freet(template_footer); \
template_footer = NULL; \
} \
hash_free(&hash); \
clearCallbacks(&opt->state.callbacks); \
} while (0)
#define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)
struct lien_buffers {
@@ -523,7 +570,6 @@ int httpmirror(char *url1, httrackp * opt) {
hash_struct *const hashptr = &hash;
t_cookie BIGSTK cookie; // gestion des cookies
//char* tab_alloc=NULL;
int ptr; // pointeur actuel sur les liens
//
@@ -534,15 +580,12 @@ int httpmirror(char *url1, httrackp * opt) {
// pour les stats, nombre de fichiers & octets écrits
LLint stat_fragment = 0; // pour la fragmentation
//TStamp istat_timestart; // départ pour calcul instantanné
//
TStamp last_info_shell = 0;
int info_shell = 0;
// filtres
char **filters = NULL;
//int filter_max=0;
int filptr = 0;
//
@@ -582,7 +625,6 @@ int httpmirror(char *url1, httrackp * opt) {
// noter heure actuelle de départ en secondes
memset(&HTS_STAT, 0, sizeof(HTS_STAT));
HTS_STAT.stat_timestart = time_local();
//istat_timestart=stat_timestart;
HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
/* reset stats */
HTS_STAT.HTS_TOTAL_RECV = 0;
@@ -616,9 +658,6 @@ int httpmirror(char *url1, httrackp * opt) {
// initialiser usercommand
usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");
// initialiser structcheck
// structcheck_init(1);
// initialiser verif_backblue
verif_backblue(opt, NULL);
verif_external(opt, 0, 0);
@@ -638,9 +677,6 @@ int httpmirror(char *url1, httrackp * opt) {
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
"templates/index-footer.html"), HTS_INDEX_FOOTER);
// initialiser mimedefs
//get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
// Initialiser indexation
if (opt->kindex)
index_init(StringBuff(opt->path_html));
@@ -684,7 +720,6 @@ int httpmirror(char *url1, httrackp * opt) {
opt->filters.filters = &filters;
//
opt->filters.filptr = &filptr;
//opt->filters.filter_max=&filter_max;
// hash table
opt->hash = &hash;
@@ -731,8 +766,7 @@ int httpmirror(char *url1, httrackp * opt) {
else if (*a == '-')
joker = 1;
if (joker) { // joker ou filters
//char* p;
if (joker) { // joker ou filters
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
int type;
int plus = 0;
@@ -787,12 +821,11 @@ int httpmirror(char *url1, httrackp * opt) {
XH_extuninit;
return 0;
}
//opt->filters.filters=filters;
}
}
} else { // adresse normale
} else { // adresse normale
char BIGSTK url[HTS_URLMAXSIZE * 2];
// prochaine adresse
@@ -867,7 +900,6 @@ int httpmirror(char *url1, httrackp * opt) {
htsbuff_cat(&primarybuff, "\n");
}
}
// fclose(fp);
hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
StringBuff(opt->filelist));
@@ -922,7 +954,6 @@ int httpmirror(char *url1, httrackp * opt) {
#endif
// backing
//soc_max=opt->maxsoc;
if (opt->maxsoc > 0) {
#if BDEBUG==2
_CLRSCR;
@@ -1236,15 +1267,10 @@ int httpmirror(char *url1, httrackp * opt) {
// tester r.adr
if (!error) {
// erreur, pas de fichier chargé:
if ((!r.adr) && (r.is_write == 0)
&& (r.statuscode != 301)
&& (r.statuscode != 302)
&& (r.statuscode != 303)
&& (r.statuscode != 307)
&& (r.statuscode != 412)
&& (r.statuscode != 416)
) {
// error=1;
if ((!r.adr) && (r.is_write == 0) && (r.statuscode != 301) &&
(r.statuscode != 302) && (r.statuscode != 303) &&
(r.statuscode != 307) && (r.statuscode != 412) &&
(r.statuscode != 416)) {
// peut être que le fichier était trop gros?
if ((istoobig
@@ -1326,30 +1352,8 @@ int httpmirror(char *url1, httrackp * opt) {
||may_be_hypertext_mime(opt, r.contenttype, urlfil())) /* Is real media, .. */
) {
/* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
//if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
// char *charset;
// char *pos;
// if (r.charset[0] != '\0') {
// charset = strdup(r.charset);
// } else {
// charset = hts_getCharsetFromMeta(r.adr, r.size);
// }
// if (charset != NULL) {
// char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
// /* Use new buffer */
// if (utf8 != NULL) {
// freet(r.adr);
// r.size = strlen(utf8);
// r.adr = utf8;
// /* New UTF-8 charset */
// r.charset[0] = '\0';
// strcpy(r.charset, "utf-8");
// }
// /* Free charset */
// free(charset);
// }
//}
/* Convert charset to UTF-8 - NOT! (what about links ? remote server
* side will have troubles with converted names) */
/* Check bogus chars */
if ((r.adr) && (r.size)) {
@@ -1490,50 +1494,6 @@ int httpmirror(char *url1, httrackp * opt) {
}
}
// MOVED IN back_finalize()
//
// --------------------
// REAL MEDIA HACK
// Check if we have to load locally the file
// --------------------
//if (!error) {
// if (r.statuscode == HTTP_OK) { // OK (ou 304 en backing)
// if (r.adr==NULL) { // Written file
// if (may_be_hypertext_mime(r.contenttype, urlfil())) { // to parse!
// LLint sz;
// sz=fsize_utf8(savename());
// if (sz>0) { // ok, exists!
// if (sz < 8192) { // ok, small file --> to parse!
// FILE* fp=FOPEN(savename(),"rb");
// if (fp) {
// r.adr=malloct(sz + 1);
// if (r.adr) {
// if (fread(r.adr,1,sz,fp) == sz) {
// r.size=sz;
// r.adr[sz] = '\0';
// r.is_write = 0;
// } else {
// freet(r.adr);
// r.size=0;
// r.adr = NULL;
// r.statuscode=STATUSCODE_INVALID;
// strcpybuff(r.msg, ".RAM read error");
// }
// fclose(fp);
// fp=NULL;
// // remove (temporary) file!
// remove(savename());
// }
// if (fp)
// fclose(fp);
// }
// }
// }
// }
// }
// }
//}
// EN OF REAL MEDIA HACK
// ---stockage en cache---
// stocker dans le cache?
@@ -1654,24 +1614,6 @@ int httpmirror(char *url1, httrackp * opt) {
}
#endif
/* info: updated */
/*
if (ptr>0) {
// "mis à jour"
if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) { // page modifiée
if (strnotempty(savename())) {
HTS_STAT.stat_updated_files++;
//if ((opt->debug>0) && (opt->log!=NULL)) {
hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
}
} else {
if (!store_errpage) {
hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
}
}
}
*/
// ------------------------------------------------------
// traitement (parsing)
// ------------------------------------------------------
@@ -1717,13 +1659,8 @@ int httpmirror(char *url1, httrackp * opt) {
free(charset);
}
/* Could not detect charset: could it be UTF-8 ? */
/* No, we can not do that: browsers do not do it
/* No, we can not do that: browsers do not do it
(and it would break links). */
//if (page_charset[0] == '\0') {
// if (is_unicode_utf8(r.adr, r.size)) {
// strcpy(page_charset, "utf-8");
// }
//}
/* Could not detect charset */
if (page_charset[0] == '\0') {
hts_log_print(opt, LOG_INFO,
@@ -1801,10 +1738,6 @@ int httpmirror(char *url1, httrackp * opt) {
XH_uninit;
return -1;
}
// I'll have to segment this part
// #include "htsparse.c"
}
}
// Fin parsing HTML
@@ -1900,7 +1833,6 @@ int httpmirror(char *url1, httrackp * opt) {
HTS_STAT.stat_files++;
HTS_STAT.stat_bytes+=r.size;
*/
//printf("ok......\n");
} else {
// Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
// récursion nous en empêche
@@ -1969,8 +1901,6 @@ int httpmirror(char *url1, httrackp * opt) {
}
}
//printf("extern=%s\n",r.contenttype);
// ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
if (r.adr != NULL || r.size == 0) {
file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
@@ -2092,16 +2022,9 @@ int httpmirror(char *url1, httrackp * opt) {
// prochain pass2
while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
ptr++;
//printf("first link==%d\n");
}
}
}
// copy abort state if necessary from outside
//if (!exit_xh && opt->state.exit_xh) {
// exit_xh=opt->state.exit_xh;
//}
// a-t-on dépassé le quota?
if (!back_checkmirror(opt)) {
ptr = opt->lien_tot;
@@ -2258,13 +2181,11 @@ int httpmirror(char *url1, httrackp * opt) {
char BIGSTK htstime[256];
char BIGSTK infoupdated[256];
// int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
LLint n =
(LLint) (HTS_STAT.HTS_TOTAL_RECV /
(max(1, time_local() - HTS_STAT.stat_timestart)));
sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
//sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
infoupdated[0] = '\0';
if (opt->is_update) {
if (HTS_STAT.stat_updated_files > 0) {
@@ -2347,12 +2268,6 @@ int httpmirror(char *url1, httrackp * opt) {
*/
int engine_stats(void) {
#if 0
static FILE *debug_fp = NULL; /* ok */
if (!debug_fp)
debug_fp = fopen("esstat.txt", "wb");
#endif
HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
HTS_STAT.nb = 0;
if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
@@ -2363,10 +2278,6 @@ int engine_stats(void) {
if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
TStamp dif;
#if 0
fprintf(debug_fp, "set timer %d\n", i);
fflush(debug_fp);
#endif
dif = cdif - HTS_STAT.istat_timestart[i];
if ((TStamp) (dif / 1000) > 0) {
LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
@@ -2385,10 +2296,6 @@ int engine_stats(void) {
// timer #0 resync timer #1 when reaching 1 second limit
if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
#if 0
fprintf(debug_fp, "resync timer 1\n");
fflush(debug_fp);
#endif
HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
HTS_STAT.istat_timestart[1] = cdif;
HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
@@ -2409,7 +2316,6 @@ void host_ban(httrackp * opt, int ptr,
lien_back *const back = sback->lnk;
const int back_max = sback->count;
//int l;
int i;
if (host[0] == '!')
@@ -2466,9 +2372,7 @@ void host_ban(httrackp * opt, int ptr,
}
// effacer liens
//l=strlen(host);
for(i = 0; i < opt->lien_tot; i++) {
//if (heap(i)->adr_len==l) { // même taille de chaîne
for (i = 0; i < opt->lien_tot; i++) {
// Calcul de taille sécurisée
if (heap(i)) {
if (heap(i)->adr) {
@@ -2507,7 +2411,6 @@ void host_ban(httrackp * opt, int ptr,
"WARNING! HostCancel detected memory leaks [null at %d]",
i);
}
//}
}
}
@@ -2924,7 +2827,6 @@ FILE *fileappend(filenote_strc * strc, const char *s) {
// noter lst
filenote(strc, s, NULL);
// if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s); // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
strcpybuff(fname, s);
#if HTS_DOSNAME
@@ -2968,7 +2870,6 @@ int filecreateempty(filenote_strc * strc, const char *filename) {
int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
// gestion du fichier liste liste
if (params) {
//filecreate_params* p = (filecreate_params*) params;
strcpybuff(strc->path, params->path);
strc->lst = params->lst;
return 0;
@@ -3048,9 +2949,8 @@ void usercommand_exe(const char *cmd, const char *file) {
}
}
static void postprocess_file(httrackp * opt, const char *save, const char *adr,
static void postprocess_file(httrackp *opt, const char *save, const char *adr,
const char *fil) {
//int first = 0;
/* MIME-html archive to build */
if (opt != NULL && opt->mimehtml) {
if (adr != NULL && strcmp(adr, "primary") == 0) {
@@ -3078,7 +2978,6 @@ static void postprocess_file(httrackp * opt, const char *save, const char *adr,
}
if (!opt->state.mimehtml_created) {
//first = 1;
opt->state.mimefp =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -3222,28 +3121,6 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
return 0;
}
// vérifier taux de transfert
#if 0
void check_rate(TStamp stat_timestart, int maxrate) {
// vérifier taux de transfert (pas trop grand?)
/*
if (maxrate>0) {
int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart)); // taux actuel de transfert
HTS_STAT.HTS_TOTAL_RECV_STATE=0;
if (r>maxrate) { // taux>taux autorisé
int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
if (taux<15)
HTS_STAT.HTS_TOTAL_RECV_STATE=1; // ralentir un peu (<15% dépassement)
else if (taux<50)
HTS_STAT.HTS_TOTAL_RECV_STATE=2; // beaucoup (<50% dépassement)
else
HTS_STAT.HTS_TOTAL_RECV_STATE=3; // énormément (>50% dépassement)
}
}
*/
}
#endif
// ---
// sous routines liées au moteur et au backing
@@ -3253,21 +3130,8 @@ int backlinks_done(const struct_back * sback,
int ptr) {
int n = 0;
#if 0
int i;
//Links done and stored in cache
for(i = ptr + 1; i < lien_tot; i++) {
if (heap(i)) {
if (heap(i)->pass2 == -1) {
n++;
}
}
}
#else
// finalized in background
n += HTS_STAT.stat_background;
#endif
n += back_done_incache(sback);
return n;
}
@@ -3404,8 +3268,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
/* on a déja parcouru */
if (p < cache->ptr_ant)
p = cache->ptr_ant;
while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
//while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
while (p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
int ok = 1;
// on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
@@ -3417,12 +3280,6 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
ok = 0;
}
// Why in hell did I do that ?
//if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0'
// && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0) // lookup in liens_record
//{
// ok = 0;
//}
if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
|| hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
ok = 0;
@@ -3451,7 +3308,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
}
}
p++;
} // while
} // while
/* sauver position dernière anticipation */
cache->ptr_ant = p;
cache->ptr_last = ptr;
@@ -3713,20 +3570,7 @@ HTSEXT_API hts_boolean hts_has_stopped(httrackp *opt) {
return ended;
}
// régler en cours de route les paramètres réglables..
// -1 : erreur
//HTSEXT_API int hts_setopt(httrackp* set_opt) {
// if (set_opt) {
// httrackp* engine_opt=hts_declareoptbuffer(NULL);
// if (engine_opt) {
// //_hts_setopt=opt;
// copy_htsopt(set_opt,engine_opt);
// }
// }
// return 0;
//}
// ajout d'URL
// -1 : erreur
HTSEXT_API hts_boolean hts_addurl(httrackp *opt, char **url) {
if (url)
opt->state._hts_addurl = url;
@@ -3987,10 +3831,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
heap_top()->link_import = 0; // pas mode import
// écrire autres paramètres de la structure-lien
//if (meme_adresse)
heap_top()->premier = heap(ptr)->premier;
//else // sinon l'objet père est le précédent lui même
// heap_top()->premier=ptr;
heap_top()->precedent = ptr;
// noter la priorité
@@ -4001,9 +3842,6 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
heap_top()->pass2 = max(pass_fix, numero_passe);
heap_top()->retry = opt->retry;
//strcpybuff(heap_top()->adr,adr);
//strcpybuff(heap_top()->fil,fil);
//strcpybuff(heap_top()->sav,save);
hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
heap_top()->adr, heap_top()->fil,
heap_top()->sav);

View File

@@ -412,10 +412,6 @@ char *readfile_utf8(const char *fil);
char *readfile_or(const char *fil, const char *defaultdata);
#if 0
void check_rate(TStamp stat_timestart, int maxrate);
#endif
/* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
Not thread-safe; call from the single crawl loop. */
@@ -467,11 +463,6 @@ int ask_continue(httrackp * opt);
/* Number of decimal digits in n. */
int nombre_digit(int n);
// Java
#if 0
int hts_add_file(char *file, int file_position);
#endif
// Polling
#if HTS_POLL
int check_flot(T_SOC s);

View File

@@ -145,7 +145,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
char *url = NULL; // URLS séparées par un espace
int url_sz = 65535;
//char url[65536]; // URLS séparées par un espace
// the parametres
int httrack_logmode = 3; // ONE log file
@@ -290,12 +289,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
if (tmp_argc == 1) { /* pas -P & co */
if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
if (argv_url < 0)
argv_url = 0; // -1==force -> 1=one url already detected, wipe all previous options
//if (argv_url>=0) {
argv_url = 0; // -1==force -> 1=one url already detected, wipe all
// previous options
argv_url++;
if (!argv_firsturl)
argv_firsturl = x_argv[x_argc - 1];
//}
} else {
if (strcmp(tmp_argv[0], "-h") == 0) {
help(argv[0], !opt->quiet);
@@ -324,8 +322,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
} else if (tmp_argc == 2) {
if ((strcmp(tmp_argv[0], "-%L") == 0)) { // liste d'URLs
if (argv_url < 0)
argv_url = 0; // -1==force -> 1=one url already detected, wipe all previous options
//if (argv_url>=0)
argv_url = 0; // -1==force -> 1=one url already detected, wipe all
// previous options
argv_url++; /* forcer */
}
}
@@ -403,15 +401,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
else
inQuote = 1;
} else if (!inQuote && !noDbl && argv[na][i] == ',') {
//StringAddchar(path, '\0');
//j = 0;
path = &opt->path_log;
} else {
StringAddchar(*path, argv[na][i]);
//path[j++] = argv[na][i];
}
}
//path[j++] = '\0';
if (StringLength(opt->path_log) == 0) {
StringCopyS(opt->path_log, opt->path_html);
}
@@ -420,7 +414,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
if (check_path(&opt->path_html, argv_firsturl)) {
opt->dir_topindex = 1; // rebuilt top index
}
//printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));
}
break;
} // switch
@@ -539,8 +532,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
insert_after++;
}
}
} while(lastp != NULL);
//fclose(fp);
} while (lastp != NULL);
}
}
@@ -716,24 +708,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
HTS_PANIC_PRINTF(s);
htsmain_free();
return -1;
#else
#if 0
char _args[8][256];
char *args[8];
printf("Cheking for updates...\n");
strcpybuff(_args[0], argv[0]);
strcpybuff(_args[1], "--get");
sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
strcpybuff(_args[3], "--quickinfo");
args[0] = _args[0];
args[1] = _args[1];
args[2] = _args[2];
args[3] = _args[3];
args[4] = NULL;
if (execvp(args[0], args) == -1) {
}
#endif
#endif
}
//
@@ -751,103 +725,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
}
}
// Compter urls/jokers
/*
if (argv_url<=0) {
int na;
argv_url=0;
for(na=1;na<argc;na++) {
if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
na++; // sauter nom de proxy
} else if (!cmdl_opt(argv[na])) {
argv_url++; // un de plus
} else if (strcmp(argv[na],"-h")==0) {
help(argv[0],!opt->quiet);
htsmain_free();
return 0;
} else {
if ((strchr(argv[na],'q')!=NULL))
opt->quiet=1; // ne pas poser de questions! (nohup par exemple)
if ((strchr(argv[na],'i')!=NULL)) { // doit.log!
argv_url=0;
na=argc;
}
}
}
}
*/
// Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
// Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
/*
if (argv_url==0) {
//if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) { // un cache est présent
x_argvblk=(char*) calloct(32768,1);
if (x_argvblk!=NULL) {
FILE* fp;
int x_argc;
//strcpybuff(x_argvblk,"httrack ");
fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
if (fp) {
linput(fp,x_argvblk+strlen(x_argvblk),8192);
fclose(fp); fp=NULL;
}
// calculer arguments selon derniers arguments
x_argv[0]=argv[0];
x_argc=1;
{
char* p=x_argvblk;
do {
x_argv[x_argc++]=p;
//p=strstr(p," ");
// exemple de chaine: "echo \"test\"" c:\a "\$0"
p=next_token(p,1); // prochain token
if (p) {
*p=0; // octet nul (tableau)
p++;
}
} while(p!=NULL);
}
// recopier arguments actuels (pointeurs uniquement)
{
int na;
for(na=1;na<argc;na++) {
if (strcmp(argv[na],"-O") != 0) // SAUF le path!
x_argv[x_argc++]=argv[na];
else
na++;
}
}
argc=x_argc; // nouvel argc
argv=x_argv; // nouvel argv
}
}
//}
}
*/
// Vérifier quiet
/*
{
int na;
for(na=1;na<argc;na++) {
if (!cmdl_opt(argv[na])) {
if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
na++; // sauter nom de proxy
} else {
if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
opt->quiet=1; // ne pas poser de questions! (nohup par exemple)
}
}
}
*/
/* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
here, ahead of the no-URL usage gate below, so they need no dummy URL. */
{
@@ -954,8 +831,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
} else { // plus de 2 paramètres
// un fichier log existe?
if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-in_progress.lock"))) { // fichier lock?
//char s[32];
StringBuff(opt->path_log),
"hts-in_progress.lock"))) { // fichier lock?
opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
if (opt->quiet == 0) {
@@ -986,11 +863,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
}
}
}
} else
if (fexist
(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
//char s[32];
} else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_html), "index.html"))) {
opt->cache = HTS_CACHE_TEST_UPDATE;
if (opt->quiet == 0) {
if ((fexist
@@ -1085,7 +959,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
//
case 'g': // récupérer un (ou plusieurs) fichiers isolés
opt->wizard = HTS_WIZARD_AUTO;
//opt->wizard=0; // pas de wizard
opt->cache = HTS_CACHE_NONE; // ni de cache
opt->makeindex = 0; // ni d'index
httrack_logmode = 1; // erreurs à l'écran
@@ -1185,9 +1058,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
while(isdigit((unsigned char) *(com + 1)))
com++;
break;
//
//case 'A': opt->urlmode=1; break;
//case 'R': opt->urlmode=2; break;
case 'K':
opt->urlmode = HTS_URLMODE_ABSOLUTE;
if (isdigit((unsigned char) *(com + 1))) {
@@ -1199,8 +1069,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
while(isdigit((unsigned char) *(com + 1)))
com++;
}
//if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
//
case 'c':
if (isdigit((unsigned char) *(com + 1))) {
sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1688,7 +1556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
} else {
char *a;
//char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
for(a = argv[na]; *a != '\0'; a++) {
if (*a == ';') { /* next one */
StringAddchar(opt->mimedefs, '\n');
@@ -2027,10 +1894,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
HTS_PANIC_PRINTF(s);
htsmain_free();
return -1;
}
break;
//case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
} break;
}
}
break;
@@ -2274,15 +2138,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
return 0;
break;
case '~': /* internal lib test */
HTS_PANIC_PRINTF
("Option #~ is disabled for security reasons");
//Disabled because choke on GCC 4.3 (toni from links2linux.de)
//{
// char thisIsATestYouShouldSeeAnError[12];
// const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
// strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
// return 0;
//}
HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
break;
case 'f':
opt->flush = 1;
@@ -2384,7 +2240,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
opt->proxy.active = 1;
// Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
a = argv[na] + strlen(argv[na]) - 1;
// a=strstr(argv[na],":"); // port
while((a > argv[na]) && (*a != ':') && (*a != '@'))
a--;
if (*a == ':') { // un port est présent, <proxy>:port
@@ -2489,7 +2344,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
#endif
#endif
//printf("WARNING! This is *only* a beta-release of HTTrack\n");
io_flush;
#if DEBUG_STEPS
@@ -2573,7 +2427,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
{
FILE *fp = NULL;
//int n=0;
char t[256];
time_local_rfc822(t); // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2607,9 +2460,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
}
strcpybuff(n_lock,
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-in_progress.lock"));
//sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-in_progress.lock"));
/*do {
if (!n)
sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2685,8 +2537,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
fprintf(fp, LF);
fclose(fp);
fp = NULL;
//} else if (opt->debug>1) {
// printf("! FileOpen error, \"%s\"\n",strerror(errno));
}
}
// petit message dans le lock

View File

@@ -102,9 +102,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
// * = toute lettre
// --?-- : spécifique à HTTrack et aux ?
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
int *size_flag) {
//int err=0;
HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
LLint *size, int *size_flag) {
if (strnotempty(joker) == 0) { // fin de chaine joker
if (strnotempty(chaine) == 0) // fin aussi pour la chaine: ok
return chaine;
@@ -236,9 +235,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
for(j = (int) (unsigned char) joker[i];
j <= (int) (unsigned char) joker[i + 2]; j++)
pass[j] = 1;
}
// else err=1;
i += 3;
} else { // 1 car, ex: *[ ]
pass[(int) (unsigned char) joker[i]] = 1;
@@ -262,7 +259,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
for(i = 0; i < 256; i++)
pass[i] = 1; // tout autoriser
jmp = 1;
////if (joker[2]==LEFT) jmp=3; // permet de recher *<crochet ouvrant>
}
{

View File

@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
#endif
// ftp mode passif
// #if HTS_INET6==0
#define FTP_PASV 1
// #else
// no passive mode for v6
// #define FTP_PASV 0
// #endif
#define FTP_DEBUG 0
//#define FORK_DEBUG 0
#if USE_BEGINTHREAD
@@ -206,20 +200,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
{
char *a;
#if 0
a = back->url_fil + strlen(back->url_fil) - 1;
while((a > back->url_fil) && (*a != '/'))
a--;
if (*a != '/') {
a = NULL;
}
#else
a = back->url_fil;
#endif
if (a != NULL && *a != '\0') {
#if 0
a++; // sauter /
#endif
ftp_filename = a;
if (strnotempty(a)) {
char catbuff[CATBUFF_SIZE];
@@ -243,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
}
} else {
strcpybuff(back->r.msg, "Unexpected PORT error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
}
@@ -277,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
snprintf(back->r.msg, sizeof(back->r.msg),
"Unable to get server's address: %s", error);
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_NON_FATAL;
_HALT_FTP return 0;
}
@@ -290,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
if (soc_ctl == INVALID_SOCKET) {
strcpybuff(back->r.msg, "Unable to create a socket");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
_HALT_FTP return 0;
}
SOCaddr_initport(server, port);
// server.sin_port = htons((unsigned short int) port);
// connexion (bloquante, on est en thread)
strcpybuff(back->info, "connect");
if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
strcpybuff(back->r.msg, "Unable to connect to the server");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
_HALT_FTP return 0;
#ifdef _WIN32
@@ -345,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
// ok
} else {
strcpybuff(back->r.msg, "TYPE I error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
#if 0
// --CWD--
char *a;
a = back->url_fil + strlen(back->url_fil) - 1;
while((a > back->url_fil) && (*a != '/'))
a--;
if (*a == '/') { // ok repéré
char BIGSTK target[1024];
target[0] = '\0';
strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
if (strnotempty(target) == 0)
strcatbuff(target, "/");
strcpybuff(back->info, "cwd");
snprintf(line, sizeof(line), "CWD %s", target);
send_line(soc_ctl, line);
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
_CHECK_HALT_FTP;
if (line[0] == '2') {
send_line(soc_ctl, "TYPE I");
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
_CHECK_HALT_FTP;
if (line[0] == '2') {
// ok..
} else {
strcpybuff(back->r.msg, "TYPE I error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
} else {
strcpybuff(back->r.msg, "Unexpected ftp error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
#endif
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
}
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
}
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
}
@@ -469,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
}
// -- fin analyse de l'adresse IP et du port --
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
} else {
@@ -501,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
}
}
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
}
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
}
@@ -613,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
deletesoc(soc_dat);
soc_dat = INVALID_SOCKET;
//
snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg),
"RETR command error: %s", linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
} else {
@@ -626,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
soc_dat = INVALID_SOCKET;
//
strcpybuff(back->r.msg, "Unable to connect");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
} else {
strcpybuff(back->r.msg, "Unable to create a socket");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
} else {
snprintf(back->r.msg, sizeof(back->r.msg),
"Unable to resolve IP %s: %s", adr_ip, error);
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
} // sinon on est prêts
#else
@@ -663,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
//T_SOC soc_dat;
if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
strcpybuff(back->r.msg, "Unable to accept connection");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg),
"RETR command error: %s", linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
}
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
}
#ifdef _WIN32
@@ -683,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
#endif
} else {
strcpybuff(back->r.msg, "Unable to listen to a port");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
#endif
@@ -707,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
int len = 1;
int read_len = 1024;
//HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues
while((len > 0) && (!stop_ftp(back))) {
// attendre les données
len = 1; // pas d'erreur pour le moment
switch (wait_socket_receive(soc_dat, timeout)) {
case -1:
strcpybuff(back->r.msg, "FTP read error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
len = 0; // fin
break;
case 0:
snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
timeout);
back->r.statuscode = STATUSCODE_INVALID;
len = 0; // fin
break;
@@ -743,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
}
*/
strcpybuff(back->r.msg, "Write error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
len = 0; // error
}
} else {
strcpybuff(back->r.msg, "Unexpected write error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
} else { // Erreur ou terminé
// back->status=STATUS_FTP_READY; // fini
} else { // Erreur ou terminé
back->r.statuscode = 0;
if (back->r.totalsize > 0
&& back->r.size != back->r.totalsize) {
@@ -762,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
}
}
read_len = 1024;
//HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues
}
}
if (back->r.fp) {
@@ -771,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
}
} else {
strcpybuff(back->r.msg, "Unable to write file");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
#ifdef _WIN32
@@ -787,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
if (line[0] == '2') { // OK
strcpybuff(back->r.msg, "OK");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = HTTP_OK;
} else {
snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
// back->status=STATUS_FTP_READY; // fini
snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
linejmp(line));
back->r.statuscode = STATUSCODE_INVALID;
}
} else {
strcpybuff(back->r.msg, "FTP read error");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
}
}
@@ -822,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
back->r.statuscode = HTTP_OK;
strcpybuff(back->r.msg, "OK");
}
// back->status=STATUS_FTP_READY; // fini
return 0;
}
@@ -989,9 +907,7 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
break;
}
//HTS_TOTAL_RECV_CHECK(dummy); // Diminuer au besoin si trop de données reçues
switch (recv(soc, &b, 1, 0)) {
//case 0: break; // pas encore --> erreur (on attend)!
case 1:
HTS_STAT.HTS_TOTAL_RECV += 1; // compter flux entrant
if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
@@ -1118,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
int stop_ftp(lien_back * back) {
if (back->stop_ftp) {
strcpybuff(back->r.msg, "Cancelled by User");
// back->status=STATUS_FTP_READY; // fini
back->r.statuscode = STATUSCODE_INVALID;
return 1;
}

View File

@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
#define str (buffers->str)
#define argv (buffers->argv)
//char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
//char *mainpath = (char *) malloct(256);
//char *projname = (char *) malloct(256);
//char *stropt = (char *) malloct(2048); // options
//char *stropt2 = (char *) malloct(2048); // options longues
//char *strwild = (char *) malloct(2048); // wildcards
//char *cmd = (char *) malloct(4096);
//char *str = (char *) malloct(256);
//char **argv = (char **) malloct(256 * sizeof(char *));
//
char *a;
@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
strcatbuff(stropt2, str);
strcatbuff(stropt2, projname);
strcatbuff(stropt2, "\" ");
// Créer si ce n'est fait un index.html 1er niveau
make_empty_index(str);
//
printf("\n");
printf("Enter URLs (separated by commas or blank spaces) :");
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
}
hts_main(argc, argv);
}
//} else {
// help("httrack",1);
}
/* Free buffers */
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
printf("Unable to create a temporary proxy (no remaining port)\n");
}
// Créer un index.html vide
void make_empty_index(const char *str) {
#if 0
if (!fexist(fconcat(str, "index.html"))) {
FILE *fp = fopen(fconcat(str, "index.html"), "wb");
if (fp) {
fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
fprintf(fp,
"<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
CRLF);
fclose(fp);
}
}
#endif
}
// mini-aide (h: help)
// y
void help(const char *app, int more) {
@@ -808,7 +777,4 @@ void help(const char *app, int more) {
infomsg("[compiled: " HTS_PLATFORM_NAME "]");
#endif
infomsg(NULL);
// infomsg(" R *relative links (e.g ../link)\n");
// infomsg(" A absolute links (e.g /www.adr/link)\n");
}

View File

@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;
void infomsg(const char *msg);
void help(const char *app, int more);
void make_empty_index(const char *str);
void help_wizard(httrackp * opt);
int help_query(const char *list, int def);
void help_catchurl(const char *dest_path);

View File

@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
char keyword[KEYW_LEN + 32];
int i = 0;
//
//int WordIndexSize = 1024;
coucal WordIndexHash = NULL;
FILE *tmpfp = NULL;
@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|| (strfield2(mime, "text/css"))
) {
inscript = 1;
//} else if (strfield2(mime, "text/vnd.wap.wml")) { // humm won't work in many cases
// inscript=0;
} else
return 0;
@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
// Process indexing for this page
{
//FILE* fp=NULL;
//fp=fopen(concat(indexpath,"index.txt"),"ab");
if (fp_tmpproject) {
while(!feof(tmpfp)) {
char line[KEYW_LEN + 32];
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
intptr_t e = 0;
if (coucal_read(WordIndexHash, line, &e)) {
//if (e) {
char BIGSTK savelst[HTS_URLMAXSIZE * 2];
e++; /* 0 means "once" */
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
fprintf(fp_tmpproject, "%s %d %s\n", line,
(int) (KEYW_SORT_MAXCOUNT - e), savelst);
hts_primindex_size++;
//}
}
}
}
//fclose(fp);
}
}
@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
off_t size = fpsize(fp_tmpproject);
if (size > 0) {
//FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
if (fp_tmpproject) {
tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
if (tab) {
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
if (total_hit) {
if (mode == 1) // TEXT
fprintf(fp, "\t=%d\r\n", total_hit);
//else // HTML
// fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
if ((((total_hit * 1000) / hts_primindex_words) >=
KEYW_USELESS1K)
|| (((total_line * 1000) / index) >=
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
if (mode == 1) // TEXT
fprintf(fp, "\t(%d)\r\n",
((total_hit * 1000) / hts_primindex_words));
//else // HTML
// fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
}
}
if (mode == 1) // TEXT

View File

@@ -175,15 +175,14 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
#if JAVADEBUG
printf("fopen\n");
#endif
if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
//fprintf(stderr, "Cannot open input file.\n");
if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
NULL) {
sprintf(str->err_msg, "Unable to open file %s", file);
return 0; // une erreur..
}
#if JAVADEBUG
printf("fread\n");
#endif
//if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) { // pas complet..
if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
fclose(fpout);
sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
@@ -257,12 +256,10 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
printf("addfiles\n");
#endif
{
//unsigned int acess;
unsigned int Class;
unsigned int SClass;
int i;
//acess = readshort(fpout);
Class = readshort(fpout);
SClass = readshort(fpout);
@@ -407,11 +404,9 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
p = &buffer[0];
//fflush(fp);
trans.file_position = ftell(fp);
length = readshort(fp);
if (length < HTS_URLMAXSIZE) {
// while ((length > 0) && (length<500)) {
while(length > 0) {
*p++ = fgetc(fp);
@@ -419,17 +414,13 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
}
*p = '\0';
//#if JDEBUG
// if(tris(buffer)==1) printf("%s\n ",buffer);
// if(tris(buffer)==2) printf("%s\n ",printname(buffer));
//#endif
if (tris(str->opt, buffer) == 1)
str->addLink(str, buffer); /* trans.file_position */
else if (tris(str->opt, buffer) == 2)
str->addLink(str, printname(rname, buffer));
strcpy(trans.name, buffer);
} else { // gros pb
} else { // gros pb
while((length > 0) && (!feof(fp))) {
fgetc(fp);
length--;
@@ -445,7 +436,6 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
}
break;
default:
// printf("Type inconnue\n");
// on arrête tout
sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
*error = 1;
@@ -508,7 +498,6 @@ static char *printname(char rname[1024], char name[1024]) {
return rname; // ""
}
p += 2;
//rname=(char*)calloct(strlen(name)+8,sizeof(char));
p1 = rname;
for(j = 0; name[j] != '\0'; j++, p++) {
if (*p == '/')

View File

@@ -598,56 +598,27 @@ static const char *hts_mime_modern[][2] = {
// Reserved (RFC2396)
#define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
#define CHAR_RESERVED(c) ( CIS(c,';') \
|| CIS(c,'/') \
|| CIS(c,'?') \
|| CIS(c,':') \
|| CIS(c,'@') \
|| CIS(c,'&') \
|| CIS(c,'=') \
|| CIS(c,'+') \
|| CIS(c,'$') \
|| CIS(c,',') )
//#define CHAR_RESERVED(c) ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
#define CHAR_RESERVED(c) \
(CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') || \
CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
// Delimiters (RFC2396)
#define CHAR_DELIM(c) ( CIS(c,'<') \
|| CIS(c,'>') \
|| CIS(c,'#') \
|| CIS(c,'%') \
|| CIS(c,'\"') )
//#define CHAR_DELIM(c) ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
#define CHAR_DELIM(c) \
(CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
// Unwise (RFC2396)
#define CHAR_UNWISE(c) ( CIS(c,'{') \
|| CIS(c,'}') \
|| CIS(c,'|') \
|| CIS(c,'\\') \
|| CIS(c,'^') \
|| CIS(c,'[') \
|| CIS(c,']') \
|| CIS(c,'`') )
//#define CHAR_UNWISE(c) ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
#define CHAR_UNWISE(c) \
(CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
// Special (escape chars) (RFC2396 + >127 )
#define CHAR_LOW(c) ( ((unsigned char)(c) <= 31) )
#define CHAR_HIG(c) ( ((unsigned char)(c) >= 127) )
#define CHAR_SPECIAL(c) ( CHAR_LOW(c) || CHAR_HIG(c) )
// We try to avoid them and encode them instead
#define CHAR_XXAVOID(c) ( CIS(c,' ') \
|| CIS(c,'*') \
|| CIS(c,'\'') \
|| CIS(c,'\"') \
|| CIS(c,'&') \
|| CIS(c,'!') )
//#define CHAR_XXAVOID(c) ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
#define CHAR_MARK(c) ( CIS(c,'-') \
|| CIS(c,'_') \
|| CIS(c,'.') \
|| CIS(c,'!') \
|| CIS(c,'~') \
|| CIS(c,'*') \
|| CIS(c,'\'') \
|| CIS(c,'(') \
|| CIS(c,')') )
//#define CHAR_MARK(c) ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
#define CHAR_XXAVOID(c) \
(CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') || \
CIS(c, '&') || CIS(c, '!'))
#define CHAR_MARK(c) \
(CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') || \
CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))
// conversion éventuelle / vers antislash
#ifdef _WIN32
@@ -841,15 +812,12 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
// treat: traiter header?
// waitconnect: attendre le connect()
// note: dans retour, on met les params du proxy
T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
const char *xsend, const char *adr, const char *fil, htsblk * retour) {
//htsblk retour;
//int bufl=TAILLE_BUFFER; // 8Ko de buffer
T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
const char *xsend, const char *adr, const char *fil,
htsblk *retour) {
T_SOC soc = INVALID_SOCKET;
char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];
//char *p,*q;
// retour prédéfini: erreur
if (retour) {
retour->adr = NULL;
@@ -934,7 +902,6 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
strcpybuff(retour->msg, "Unable to open local file");
else {
// Note: On passe par un FILE* (plus propre)
//soc=open(fil,O_RDONLY,0); // en lecture seule!
retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb"); // ouvrir
if (retour->fp == NULL)
@@ -1011,17 +978,10 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
} while(strnotempty(rcvd));
//rcvsize=-1; // forCER CHARGEMENT INCONNU
//if (retour)
// retour->totalsize=rcvsize;
} else { // si GET, on recevra l'en tête APRES
//rcvsize=-1; // on ne connait pas la taille de l'en-tête
} else { // si GET, on recevra l'en tête APRES
if (retour)
retour->totalsize = -1;
}
}
}
@@ -1118,19 +1078,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
char BIGSTK buffer_head_request[16384];
buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };
//int use_11=0; // HTTP 1.1 utilisé
int direct_url = 0; // ne pas analyser l'url (exemple: ftp://)
const char *search_tag = NULL;
// Initialize buffer
buffer_head_request[0] = '\0';
// header Date
//strcatbuff(buff,"Date: ");
//time_gmt_rfc822(buff); // obtenir l'heure au format rfc822
//sendc("\n");
//strcatbuff(buff,buff);
// possibilité non documentée: >post: et >postfile:
// si présence d'un tag >post: alors executer un POST
// exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
@@ -1224,11 +1177,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
}
// protocole
if (!retour->req.http11) { // forcer HTTP/1.0
//use_11=0;
if (!retour->req.http11) { // forcer HTTP/1.0
print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
} else { // Requète 1.1
//use_11=1;
} else { // Requète 1.1
print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
}
@@ -1364,9 +1315,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
}
}
}
//strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");
// Custom header(s)
if (strnotempty(retour->req.headers)) {
@@ -1520,14 +1469,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
a += strlen("filename=");
while(is_space(*a))
a++;
//a=strchr(a,'"');
if (a) {
char *c = NULL;
//a++; /* jump " */
while((c = strchr(a, '/'))) /* skip all / (see RFC2616) */
a = c + 1;
//b=strchr(a+1,'"');
b = a + strlen(a) - 1;
while(is_space(*b))
b--;
@@ -1544,16 +1490,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
} else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
while(is_realspace(*(rcvd + p)))
p++; // sauter espaces
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
//struct tm* tm_time=convert_time_rfc822(rcvd+p);
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
strcpybuff(retour->lastmodified, rcvd + p);
}
} else if ((p = strfield(rcvd, "Date:")) != 0) {
if (strnotempty(retour->lastmodified) == 0) { /* pas encore de last-modified */
while(is_realspace(*(rcvd + p)))
p++; // sauter espaces
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
//struct tm* tm_time=convert_time_rfc822(rcvd+p);
if ((int) strlen(rcvd + p) < 64) { // pas trop long?
strcpybuff(retour->lastmodified, rcvd + p);
}
}
@@ -1566,14 +1510,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
else // erreur.. ignorer
retour->etag[0] = '\0';
}
}
// else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) { // chunk!
else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
} else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
while(is_realspace(*(rcvd + p)))
p++; // sauter espaces
if (strfield(rcvd + p, "chunked")) {
retour->is_chunk = 1; // chunked
//retour->http11=2; // chunked
retour->is_chunk = 1; // chunked
#if HDEBUG
printf("ok, Transfer-Encoding: détecté\n");
#endif
@@ -1750,7 +1691,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
retour->location[0] = '\0';
}
}
} else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) { // ohh un cookie
} else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
(cookie)) { // ohh un cookie
char *a = rcvd + p; // pointeur
char domain[256]; // domaine cookie (.netscape.com)
char path[256]; // chemin (/)
@@ -1794,10 +1736,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
a++; // sauter espaces
value_st = a;
while((*a != ';') && (*a))
a++; // prochain ;
//while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++; // prochain " (et pas \")
a++; // prochain ;
value_end = a;
//if (*a==';') { // finit par un ;
// vérifier débordements
if ((((int) (token_end - token_st)) < 200)
&& (((int) (value_end - value_st)) < 8000)
@@ -2139,8 +2079,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
nl = READ_ERROR;
}
}
//if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
// nl=-1; // break
// libérer bloc tempo
freet(buff);
@@ -2228,9 +2166,6 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
T_SOC soc;
htsblk retour;
//int rcvsize=-1;
//char* rcv=NULL; // adresse de retour
//int bufl=TAILLE_BUFFER; // 8Ko de buffer
TStamp tl;
int timeout = 30; // timeout pour un check (arbitraire) // **
@@ -2239,11 +2174,8 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
loc[0] = '\0';
hts_init_htsblk(&retour);
//memset(&retour, 0, sizeof(htsblk)); // effacer
retour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx
//soc=http_fopen(adr,fil,&retour,NULL); // ouvrir, + header
// on ouvre en head, et on traite l'en tête
soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour); // ouvrir HEAD, + envoi header
@@ -2619,7 +2551,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
// chemin www... trop long!!
if ((((int) (q - p))) > HTS_URLMAXSIZE) {
//strcpybuff(retour.msg,"Path too long");
return -1; // erreur
}
// recopier adrfil->adresse www..
@@ -2805,7 +2736,6 @@ void deletesoc_r(htsblk * r) {
#if HTS_USEOPENSSL
if (r->ssl_con) {
SSL_shutdown(r->ssl_con);
// SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
SSL_free(r->ssl_con);
r->ssl_con = NULL;
}
@@ -3194,7 +3124,6 @@ int finput(T_SOC fd, char *s, int max) {
int j = 0;
do {
//c=fgetc(fp);
if (read((int) fd, &c, 1) <= 0) {
c = 0;
}
@@ -3210,7 +3139,7 @@ int finput(T_SOC fd, char *s, int max) {
break;
}
}
} while((c != 0) && (j < max - 1));
} while ((c != 0) && (j < max - 1));
s[j] = '\0';
return j;
}
@@ -4869,7 +4798,6 @@ LLint check_downloadable_bytes(int rate) {
time_now = mtime_local();
elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
// NO totally stupid - elapsed_useconds+=1000; // for the next second, too
bytes_transferred_during_period =
(HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);
@@ -4882,32 +4810,6 @@ LLint check_downloadable_bytes(int rate) {
return TAILLE_BUFFER;
}
//
// 0 : OK
// 1 : slow down
#if 0
int HTS_TOTAL_RECV_CHECK(int var) {
if (HTS_STAT.HTS_TOTAL_RECV_STATE)
return 1;
/*
{
if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) {
var = min(var,32);
Sleep(250);
} else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) {
var = min(var,256);
Sleep(100);
} else {
var/=2;
if (var<=0) var=1;
Sleep(50);
}
}
*/
return 0;
}
#endif
// Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
// returns:
// >0 : data received
@@ -4916,7 +4818,6 @@ int HTS_TOTAL_RECV_CHECK(int var) {
int hts_read(htsblk * r, char *buff, int size) {
int retour;
// return read(soc,buff,size);
if (r->is_file) {
#if HTS_WIDE_DEBUG
DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
@@ -4934,7 +4835,6 @@ int hts_read(htsblk * r, char *buff, int size) {
if (r->soc == INVALID_SOCKET)
printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
#endif
//HTS_TOTAL_RECV_CHECK(size); // Diminuer au besoin si trop de données reçues
#if HTS_USEOPENSSL
if (r->ssl) {
retour = SSL_read(r->ssl_con, buff, size);
@@ -5369,18 +5269,13 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)
// --- Tracage des mallocs() ---
#ifdef HTS_TRACE_MALLOC
//#define htsLocker(A, N) htsLocker(A, N)
#define htsLocker(A, N) do {} while(0)
static mlink trmalloc = { NULL, 0, 0, NULL };
static int trmalloc_id = 0;
static htsmutex *mallocMutex = NULL;
static void hts_meminit(void) {
//if (mallocMutex == NULL) {
// mallocMutex = calloc(sizeof(*mallocMutex), 1);
// htsLocker(mallocMutex, -999);
//}
}
static void hts_meminit(void) {}
void *hts_malloc(size_t len) {
void *adr;
@@ -5464,7 +5359,6 @@ void hts_free(void *adr) {
htsboundary);
lnk->next = lnk->next->next;
free((void *) blk_free);
//blk_free->id=-1;
free((char *) adr - bsize);
htsLocker(mallocMutex, 0);
return;
@@ -5533,7 +5427,6 @@ mlink *hts_find(char *adr) {
if (depl < 0)
depl = -depl;
//assertf(depl < 512000); /* near the stack frame.. doesn't look like malloc but stack variable */
return NULL;
}
}
@@ -5589,8 +5482,7 @@ int ftp_available(void) {
}
#else
int ftp_available(void) {
return 1; // ok!
//return 0; // SOUS UNIX, PROBLEMESs
return 1; // ok!
}
#endif
@@ -5804,7 +5696,6 @@ HTSEXT_API int hts_init(void) {
assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
}
// OpenSSL_add_all_algorithms();
openssl_ctx = SSL_CTX_new(method);
if (!openssl_ctx) {
fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
@@ -6126,7 +6017,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
opt->log = stdout;
opt->errlog = stderr;
opt->flush = HTS_TRUE;
// opt->aff_progress=0;
opt->keyboard = HTS_FALSE;
//
StringCopy(opt->path_html, "");
@@ -6136,8 +6026,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
//
opt->maxlink = 100000; // 100,000 liens max par défaut
opt->maxfilter = 200; // 200 filtres max par défaut
opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
//opt->maxcache_anticipate=256; // maximum de liens à anticiper
opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
// PARAMETRABLE POUR L'INSTANT --
opt->maxtime = -1; // temps max en secondes
opt->maxrate = 100000; // taux maxi
opt->maxconn = 5.0; // nombre connexions/s
@@ -6290,11 +6180,9 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
}
// defaut wrappers
static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
}
static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
// hts_freevar();
}
static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
return 1;
}

View File

@@ -171,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
// fonctions unix/winsock
int hts_read(htsblk * r, char *buff, int size);
//int HTS_TOTAL_RECV_CHECK(int var);
LLint check_downloadable_bytes(int rate);
HTSEXT_API int hts_uninit_module(void);
@@ -190,7 +189,6 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
const char *path, char *dst, size_t dst_size);
//int newhttp(char* iadr,char* err=NULL);
T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
int waitconnect);
/* Like newhttp(), but connect to the addr_index-th resolved address of the host

View File

@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
/* >>> Put all modules variables here */
#if 0
t_gzopen gzopen = NULL;
t_gzread gzread = NULL;
t_gzclose gzclose = NULL;
#endif
int V6_is_available = HTS_INET6;
static char WHAT_is_available[64] = "";
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
return -1;
}
//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
// while(chain->next != NULL) {
// chain = chain->next;
// }
// chain->next = calloct(1, sizeof(htscallbacks));
// assertf(chain->next != NULL);
// chain = chain->next;
// memset(chain, 0, sizeof(*chain));
// chain->exitFnc = exitFnc;
// chain->moduleHandle = moduleHandle;
//}
void clearCallbacks(htscallbacks * chain_);
void clearCallbacks(htscallbacks * chain_) {
htscallbacks *chain;

View File

@@ -1332,8 +1332,7 @@ int url_savename(lien_adrfilsave *const afs,
while((a > afs->save) && (*a != '.') && (*a != '/'))
a--;
if (*a != '.') { // agh pas de point
//strcatbuff(save,".none"); // a éviter
if (*a != '.') { // agh pas de point
strcatbuff(afs->save, ".html"); // préférable!
hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
adr_complete, fil_complete, afs->save);
@@ -1471,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
fil_simplifie(afs->save);
/* convert name to UTF-8 ? Note: already done while parsing. */
//if (charset != NULL && charset[0] != '\0') {
// char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
// if (s != NULL) {
// hts_log_print(opt, LOG_DEBUG,
// "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
// save, s, charset);
// strcpybuff(save, s);
// free(s);
// }
//}
/* callback */
RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1629,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
int sameFil;
// NO - URL hack is only for stripping // and www.
//if (opt->urlhack != 0)
// sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
//else
sameFil = (strcmp(heap(i)->fil, normfil) == 0);
if (sameAdr && sameFil) { // ok c'est le même lien, adresse déja définie
/* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1698,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,
strcpybuff(afs->save, tempo);
//printf("switched: %s\n",save);
} // if
}
#if DEBUG_SAVENAME
printf("\nEnd search, %s\n", fil_complete);
#endif
} while(!nom_ok);
} while (!nom_ok);
}
//printf("'%s' %s %s\n",save,adr,fil);
return 0;
}

View File

@@ -105,7 +105,6 @@ typedef struct htsfilters htsfilters;
struct htsfilters {
char ***filters; /**< pointer to the +/-pattern filter array */
int *filptr; /**< pointer to the current filter count */
// int* filter_max;
};
/* User callbacks chain */
@@ -440,7 +439,6 @@ struct httrackp {
float maxconn; /**< max connections per second */
int waittime; /**< scheduled start time (wall-clock seconds) */
hts_cachemode cache; /**< cache generation mode */
// int aff_progress; // progress bar
hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
t_proxy proxy; /**< proxy configuration */
hts_savename_83
@@ -492,7 +490,6 @@ struct httrackp {
hts_verbosedisplay verbosedisplay; /**< animated text progress display */
String footer; /**< footer/info line injected into pages */
int maxcache; /**< in-memory cache backing limit (bytes) */
// int maxcache_anticipate; // maximum links to anticipate (upper bound)
hts_boolean ftp_proxy; /**< use the HTTP proxy for FTP too */
String filelist; /**< file listing URLs to include */
String urllist; /**< file listing filters to include */
@@ -549,8 +546,6 @@ typedef struct hts_stat_struct hts_stat_struct;
struct hts_stat_struct {
LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
LLint stat_bytes; /**< total bytes written to disk */
// int HTS_TOTAL_RECV_STATE; // status: 0 ok 1: slow down a little 2: slow
// down 3: a lot
TStamp stat_timestart; /**< mirror start time */
//
LLint total_packed; /**< compressed bytes received (on the wire) */
@@ -682,7 +677,6 @@ struct lien_url {
int depth; /**< remaining allowed depth; >0 strong, 0 weak */
int pass2; /**< second-pass marker; -1 means handled in background */
char link_import; /**< imported after a move; skip the usual up/down rules */
// int moved; // pointer to moved
int retry; /**< remaining retries */
int testmode; /**< test only: send just a HEAD */
};
@@ -727,9 +721,6 @@ struct lien_back {
LLint chunk_size; /**< size of the chunk being loaded */
LLint chunk_blocksize; /**< data size declared by the chunk */
LLint compressed_size; /**< compressed size (stats only) */
//
// int links_index; // to access liens[links_index]
//
char info[256]; /**< status text, e.g. for FTP */
int stop_ftp; /**< stop flag for FTP */
int finalized; /**< finalized (memory optimization) */

View File

@@ -395,16 +395,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
// terminaison (" ou ') du "<body onLoad=.."
int inscriptgen = 0; // on est dans un code générant, ex après obj.write("..
//int inscript_check_comments=0, inscript_in_comments=0; // javascript comments
char scriptgen_q = '\0'; // caractère faisant office de guillemet (' ou ")
//int no_esc_utf=0; // ne pas echapper chars > 127
int nofollow = 0; // ne pas scanner
//
int parseall_lastc = '\0'; // dernier caractère parsé pour parseall
//int parseall_incomment=0; // dans un /* */ (exemple: a = /* URL */ "img.gif";)
//
const char *intag_start = html;
const char *intag_name = NULL;
@@ -517,12 +514,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
lastsaved = html;
}
}
// Detect UTF8 format
//if (is_unicode_utf8(r->adr, (unsigned int) r->size) == 1) {
// no_esc_utf=1;
//} else {
// no_esc_utf=0;
//}
// Hack to prevent any problems with ram files of other files
*(r->adr + r->size) = '\0';
@@ -687,8 +678,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
) {
intag = 1;
intag_ctype = 0;
//parseall_incomment=0;
//inquote=0; // effacer quote
intag_start = html;
for(intag_name = html + 1; is_realspace(*intag_name); intag_name++) ;
intag_start_valid = 1;
@@ -706,8 +695,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
if (len > 0) {
if (strfield(token, "content-type")) {
intag_ctype = 1;
//NOPE-we do not convert the whole page actually
//intag_start[1] = 'X';
} else if (strfield(token, "refresh")) {
intag_ctype = 2;
}
@@ -768,7 +755,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil)), gmttime,
HTTRACK_VERSIONID, /* EOF */ NULL);
strcatbuff(tempo, eol);
//fwrite(tempo,1,strlen(tempo),fp);
HT_ADD(tempo);
}
// Emit charset ?
@@ -809,8 +795,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
intag = 0; //inquote=0;
// entrée dans du javascript?
// on parse ICI car il se peut qu'on ait eu a parser les src=.. dedans
//if (!inscript) { // sinon on est dans un obj.write("..
// on parse ICI car il se peut qu'on ait eu a parser les src=..
// dedans
if ((intag_start_valid) && (check_tag(intag_start, "script")
|| check_tag(intag_start, "style")
)
@@ -856,12 +842,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
#endif
}
//}
}
//else if (*adr==34) {
// inquote=(inquote?0:1);
//}
else if (intag || inscript || in_media) { // nous sommes dans un tag/commentaire, tester si on recoit un tag
} else if (intag || inscript ||
in_media) { // nous sommes dans un tag/commentaire, tester si
// on recoit un tag
int p_type = 0;
int p_nocatch = 0;
int p_searchMETAURL = 0; // chercher ..URL=<url>
@@ -1100,27 +1083,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
if (p) {
if (intag_ctype == 1) {
p = 0;
#if 0
//if ((pos=rech_tageq(html, "content"))) {
char temp[256];
char *token = NULL;
int len = rech_endtoken(html + pos, &token);
if (len > 0 && len < sizeof(temp) - 2) {
char *chpos;
temp[0] = '\0';
strncat(temp, token, len);
if ((chpos = strstr(temp, "charset"))
&& (chpos = strchr(chpos, '='))
) {
chpos++;
while(is_space(*chpos))
chpod++;
//chpos
}
}
#endif
}
// <META HTTP-EQUIV="Refresh" CONTENT="3;URL=http://www.example.com">
else if (intag_ctype == 2) {
@@ -1305,21 +1267,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
} else if (inscript) {
#if 0
/* Check // javascript comments */
if (*html == 10 || *html == 13) {
inscript_check_comments = 1;
inscript_in_comments = 0;
} else if (inscript_check_comments) {
if (!is_realspace(*html)) {
inscript_check_comments = 0;
if (html[0] == '/' && html[1] == '/') {
inscript_in_comments = 1;
}
}
}
#endif
/* Parse */
assertf(inscript_name != NULL);
if (*html == '/'
@@ -1332,7 +1279,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
&& inscript_locked == 0) {
const char *a = html;
//while(is_realspace(*(--a)));
while(is_realspace(*a))
a--;
a--;
@@ -1343,8 +1289,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
HT_ADD("<@@ /inscript @@>");
}
}
} else if (inscript_state_pos ==
INSCRIPT_START /*!inscript_in_comments */ ) {
} else if (inscript_state_pos == INSCRIPT_START) {
/*
Script Analyzing - different types supported:
foo="url"
@@ -1533,12 +1478,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
} else { // ptr == 0
//p=rech_tageq(adr,"primary"); // lien primaire, yeah
p = 0; // No stupid tag anymore, raw link
valid_p = 1; // Valid even if p==0
while((html[p] == '\r') || (html[p] == '\n'))
p++;
//can_avoid_quotes=1;
ending_p = '\r';
}
@@ -1551,25 +1494,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
// risque: générer de faux fichiers parazites
// fix: ne parse plus dans les commentaires
// ------------------------------------------------------------
if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 && (ptr > 0) && (!in_media) /* && (!inscript_in_comments) */ ) { // option parsing "brut"
//int incomment_justquit=0;
if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 &&
(ptr > 0) && (!in_media)) { // option parsing "brut"
if (!is_realspace(*html)) {
int noparse = 0;
// Gestion des /* */
#if 0
if (inscript) {
if (parseall_incomment) {
if ((*html == '/') && (*(html - 1) == '*'))
parseall_incomment = 0;
incomment_justquit = 1; // ne pas noter dernier caractère
} else {
if ((*html == '/') && (*(html + 1) == '*'))
parseall_incomment = 1;
}
} else
parseall_incomment = 0;
#endif
/* ensure automate state 0 (not in comments, quotes..) */
if (inscript
&& (inscript_state_pos != INSCRIPT_INQUOTE
@@ -1583,7 +1512,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
// recherche d'URLs
if (!noparse) {
//if ((!parseall_incomment) && (!noparse)) {
if (!p) { // non déja trouvé
if (html != r->adr) { // >1 caractère
// scanner les chaines
@@ -1607,9 +1535,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
&& (count > 0)) {
char c;
//char* aend;
//
//aend=a; // sauver début
a++;
while(is_taborspace(*a))
a++;
@@ -1664,10 +1589,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
if (!invalid_url) {
// Un plus à la fin? Alors ne pas prendre sauf si extension ("/toto.html#"+tag)
if (c != '+') { // PAS de plus à la fin
#if 0
char *a;
#endif
// "Comparisons of scheme names MUST be case-insensitive" (RFC2616)
// "Comparisons of scheme names MUST be
// case-insensitive" (RFC2616)
if ((strfield(tempo, "http:"))
|| (strfield(tempo, "ftp:"))
#if HTS_USEOPENSSL
@@ -1680,15 +1603,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
if (inscript) // sinon si pas javascript, méfiance (répertoire style base?)
url_ok = 1;
}
#if 0
else if ((a = strchr(tempo, '/'))) { // un slash: ok..
if (inscript) { // sinon si pas javascript, méfiance (style "text/css")
if (strchr(a + 1, '/')) // un seul / : abandon (STYLE type='text/css')
if (!strchr(tempo, ' ')) // avoid spaces (too dangerous for comments)
url_ok = 1;
}
}
#endif
}
// Prendre si extension reconnue
if (!url_ok) {
@@ -1767,9 +1681,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
}
}
} // p == 0
} // p == 0
} // not in comment
} // not in comment
// plus dans un commentaire
if (inscript_state_pos == INSCRIPT_START
@@ -1778,17 +1692,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
} // if realspace
} // if parseall
} // if parseall
// ------------------------------------------------------------
// p!=0 : on a repéré un éventuel lien
// ------------------------------------------------------------
//
if ((p > 0) || (valid_p)) { // on a repéré un lien
//int lien_valide=0;
if ((p > 0) || (valid_p)) { // on a repéré un lien
const char *eadr = NULL; /* fin de l'URL */
//char* quote_adr=NULL; /* adresse du ? dans l'adresse */
int ok = 1;
char quote = '\0';
int quoteinscript = 0;
@@ -1811,7 +1723,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
lastsaved = html; // dernier écrit+1
}
// sauter espaces
// adr+=p;
hts_automate_increment(&saut, p);
while((is_space(*html)
|| (inscriptgen && html[0] == '\\' && is_space(html[1])
@@ -1840,8 +1751,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
// sauter éventuel \" ou \' javascript
if (inscript) { // on est dans un obj.write("..
if (*html == '\\') {
if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
// html+=2; // sauter
if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
hts_automate_increment(&saut, 2);
}
}
@@ -1916,7 +1826,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
ok = -1; // ne pas traiter ce lien
if (ok > 0) {
//if (*eadr!=' ') {
if (is_space(*eadr)) { // guillemets,CR, etc
if ((*eadr == quote && (!quoteinscript || *(eadr - 1) == '\\')) // end quote
|| (noquote && (*eadr == '\"' || *eadr == '\'')) // end at any quote
@@ -1962,7 +1871,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
break;
}
}
//}
}
eadr++;
} while(ok == 1);
@@ -1991,14 +1899,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
char BIGSTK lien[HTS_URLMAXSIZE * 2];
int meme_adresse = 0; // 0 par défaut pour primary
//char *copie_de_adr=html;
//char* p;
// construire lien (découpage)
if (eadr - html - 1 < HTS_URLMAXSIZE) { // pas trop long?
strncpy(lien, html, eadr - html - 1);
lien[eadr - html - 1] = '\0';
//printf("link: %s\n",lien);
// supprimer les espaces
while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
lien[strlen(lien) - 1] = '\0';
@@ -2242,7 +2146,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
a += 2;
else
a = lien;
// while((*a) && (*a!='/') && (*a!=':')) a++;
a = jump_toport(a);
if (a) { // port
int port = 0;
@@ -2250,9 +2153,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
char *b = a + 1;
#if HTS_USEOPENSSL
// FIXME
//if (strfield(adr, "https:")) {
//}
#endif
while(isdigit((unsigned char) *b)) {
port *= 10;
@@ -2343,13 +2243,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
*(a + 1) = '\0';
}
}
//char BIGSTK tempo[HTS_URLMAXSIZE*2];
//strcpybuff(tempo,"http://");
//strcatbuff(tempo,urladr()); // host
//if (*lien!='/')
// strcatbuff(tempo,"/");
//strcatbuff(tempo,lien);
//strcpybuff(lien,tempo);
}
}
@@ -2378,22 +2271,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
// stocker base ou codebase?
switch (p_type) {
case 2:{
//if (*lien!='/') strcatbuff(base,"/");
strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
}
break; // base
case -2:{
//if (*lien!='/') strcatbuff(codebase,"/");
strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
}
break; // base
case 2: {
strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
} break; // base
case -2: {
strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
} break; // base
}
hts_log_print(opt, LOG_DEBUG,
"code/codebase link %s base %s", lien,
base);
//printf("base code: %s - %s\n",lien,base);
}
} else {
@@ -2506,7 +2394,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
// Tester si un lien doit être accepté ou refusé (wizard)
// forbidden_url=1 : lien refusé
// forbidden_url=0 : lien accepté
//if ((ptr>0) && (p_type!=2) && (p_type!=-2)) { // tester autorisations?
if ((p_type != 2) && (p_type != -2)) { // tester autorisations?
if (!p_nocatch) {
if (afs.af.adr[0] != '\0') {
@@ -2556,9 +2443,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
/* Calc */
last_adr[0] = '\0';
//char last_fil[HTS_URLMAXSIZE*2]="";
strcpybuff(last_adr, afs.af.adr); // ancienne adresse
//strcpybuff(last_fil,fil); // ancien chemin
strcpybuff(last_adr, afs.af.adr); // ancienne adresse
r_sv =
url_savename(&afs, &former, heap(ptr)->adr, heap(ptr)->fil, opt,
sback, cache, hash, ptr,
@@ -2589,7 +2474,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
}
}
//import_done=1; // c'est un import!
meme_adresse = 0; // on a changé
}
} else {
@@ -2787,11 +2671,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
if (lienrelatif(tempo, sizeof(tempo), save,
relativesavename()) == 0) {
/* Never escape high-chars (we don't know the encoding!!) */
inplace_escape_uri_utf(tempo, sizeof(tempo)); // escape with %xx
//if (!no_esc_utf)
// escape_uri(tempo); // escape with %xx
//else
// escape_uri_utf(tempo); // escape with %xx
inplace_escape_uri_utf(
tempo, sizeof(tempo)); // escape with %xx
HT_ADD_HTMLESCAPED(tempo); // page externe
if (add_url) {
HT_ADD("?link="); // page externe
@@ -3028,16 +2909,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
// unquoted url() (CSS/JS): keep parens escaped
if (ending_p == ')')
escape_url_parens(tempo, sizeof(tempo));
//if (!no_esc_utf)
// escape_uri(tempo); // escape with %xx
//else {
// /* No escaping at all - remaining upper chars will be escaped below */
// /* FIXME - Should be done in all local cases */
// //x_escape_html(tempo);
// //escape_uri_utf(tempo); // FIXME - escape with %xx
// //escape_uri(tempo); // escape with %xx
//}
}
hts_log_print(opt, LOG_DEBUG,
"relative link at %s build with %s and %s: %s",
@@ -3095,22 +2966,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
HT_ADD(tempo4); // refresh code="
}
}
//lastsaved=adr; // dernier écrit+1
}
if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
// convert to local codepage - NOT, already converted into %NN, and passed to the remote server so we do not have anything to do
//if (str->page_charset_ != NULL && *str->page_charset_ != '\0') {
// char *const local_save = hts_convertStringFromUTF8(tempo, strlen(tempo), str->page_charset_);
// if (local_save != NULL) {
// strcpybuff(tempo, local_save);
// free(local_save);
// } else {
// if ((opt->debug>1) && (opt->log!=NULL)) {
// fprintf(opt->log, "Warning: could not build local charset representation of '%s' in '%s'"LF, tempo, str->page_charset_);
// }
// }
//}
// écrire le lien modifié, relatif
// Note: escape all chars, even >127 (no UTF)
@@ -3134,16 +2992,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
}
} // sinon le lien sera écrit normalement
#if 0
if (fexist(save)) { // le fichier existe..
adr[0] = '\0';
//if ((opt->debug>0) && (opt->log!=NULL)) {
hts_log_print(opt, LOG_WARNING,
"Link has already been written on disk, cancelled: %s",
save);
}
#endif
/* Security check */
if (strlen(afs.save) >= HTS_URLMAXSIZE) {
afs.af.adr[0] = '\0';
@@ -3185,9 +3033,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
/*
if (strnotempty(save)) {
if (ishtml(opt,save) == 1) {
// descore_prio = 2;
} else {
// descore_prio = 1;
}
}
*/
@@ -3223,8 +3069,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
// >>>> CREER LE LIEN <<<<
//
// enregistrer lien à charger
//heap_top()->adr[0]=heap_top()->fil[0]=heap_top()->sav[0]='\0';
// même adresse: l'objet père est l'objet père de l'actuel
// DEBUT ROBOTS.TXT AJOUT
if (!just_test_it) {
@@ -3303,7 +3147,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
heap_top()->premier = heap(ptr)->premier;
else // sinon l'objet père est le précédent lui même
heap_top()->premier = heap_top_index();
// heap_top()->premier=ptr;
heap_top()->precedent = ptr;
// noter la priorité
@@ -3315,9 +3158,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
heap_top()->pass2 = pass_fix;
heap_top()->retry = opt->retry;
//strcpybuff(heap_top()->adr,adr);
//strcpybuff(heap_top()->fil,fil);
//strcpybuff(heap_top()->sav,save);
if (!just_test_it) {
hts_log_print(opt, LOG_DEBUG,
"OK, NOTE: %s%s -> %s",
@@ -3352,7 +3192,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
if (eadr > html) {
hts_automate_increment(&saut, (int) (eadr - 1 - html));
}
// adr=eadr-1; // ** sauter
/* srcset candidate loop: skip the descriptor and comma, then
re-enter the capture for the next URL. Backward goto, not a loop:
@@ -3382,9 +3221,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
inscript_state_pos = INSCRIPT_START;
} */
} // if (p)
} // if (p)
} // si '<' ou '>'
} // si '<' ou '>'
// plus loin
html++; // automate will be checked next loop
@@ -3454,9 +3293,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
*stre->exit_xh_ = 1; // exit requested
XH_uninit;
return -1;
//adr = r->adr + r->size; // exit
} else if (opt->state._hts_cancel == 1) {
// adr = r->adr + r->size; // exit
nofollow = 1; // moins violent
opt->state._hts_cancel = 0;
}
@@ -3514,9 +3351,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
fp = NULL;
}
}
// sauver fichier
//structcheck(savename());
//filesave(opt,r->adr,r->size,savename());
} // analyse OK
@@ -3555,17 +3389,11 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
// DEBUT rattrapage des 301,302,307..
// ------------------------------------------------------------
if (!error) {
////////{
// on a chargé un fichier en plus
// if (!error) stat_loaded+=r.size;
// ------------------------------------------------------------
// Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing
// ------------------------------------------------------------
if (HTTP_IS_REDIRECT(r->statuscode)) {
//if (r->adr!=NULL) { // adr==null si fichier direct. [catch: davename normalement si cgi]
//int i=0;
// char* p;
hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, urladr(), urlfil());
@@ -3592,7 +3420,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
// A same-file alias redirect must be followed, not stubbed (#159).
const hts_boolean same_savefile = hts_redirect_same_savefile(
opt, urladr(), urlfil(), moved->adr, moved->fil);
//if (ident_url_absolute(mov_url,moved->adr,moved->fil)!=-1) { // ok URL reconnue
// c'est (en gros) la même URL..
// si c'est un problème de casse dans le host c'est que le serveur est buggé
// ("RFC says.." : host name IS case insensitive)
@@ -3632,7 +3459,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
} /* sinon traité normalement */
}
//if ((strfield2(moved->adr,urladr())!=0) && (strfield2(moved->fil,urlfil())!=0)) { // identique à casse près
if (get_it == 1) {
// court-circuiter le reste du traitement
// et reculer pour mieux sauter
@@ -3643,9 +3469,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
error = 1;
hts_invalidate_link(opt, ptr); // invalidate hashtable entry
// noter NOUVEAU lien
//xxc xxc
// set_prio_to=0+1; // protection if the moved URL is an html page!!
//xxc xxc
{
// calculer lien et éventuellement modifier addresse/fichier
if (url_savename(&savedmoved, NULL,
@@ -3683,8 +3506,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
}
}
//printf("-> %s %s %s\n",liens[lien_tot-1]->adr,liens[lien_tot-1]->fil,liens[lien_tot-1]->sav);
// note métaphysique: il se peut qu'il y ait un index.html et un INDEX.HTML
// sous DOS ca marche pas très bien... mais comme je suis génial url_savename()
// est à même de régler ce problème
@@ -3806,7 +3627,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
// cas où l'on peut reessayer
switch (r->statuscode) {
//case -1: can_retry=1; break;
case STATUSCODE_TIMEOUT:
if (opt->hostcontrol) { // timeout et retry épuisés
if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) &&
@@ -3938,7 +3758,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
// FIN rattrapage des 301,302,307..
// ------------------------------------------------------------
} // if !error
} // if !error
/* Apply changes */
ENGINE_SAVE_CONTEXT();
@@ -4005,9 +3825,6 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
}
}
}
// On désalloue le buffer d'enregistrement des chemins créée, au cas où pendant la pause
// l'utilisateur ferait un rm -r après avoir effectué un tar
// structcheck_init(1);
{
FILE *fp =
fopen(fconcat
@@ -4388,33 +4205,8 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
"link #%d is ready, no more on the stack, skipping: %s%s..",
ptr, urladr(), urlfil());
// prochain lien
// ptr++;
return 2; // goto jump_if_done;
return 2; // goto jump_if_done;
}
#if 0
/* FIXME - finalized HAS NO MORE THIS MEANING */
/* link put in cache by the backing system for memory spare - reclaim */
else if (back[b].finalized) {
assertf(back[b].r.adr == NULL);
/* read file in cache */
back[b].r =
cache_read_ro(opt, cache, back[b].url_adr, back[b].url_fil,
back[b].url_sav, back[b].location_buffer);
/* ensure correct location buffer set */
back[b].r.location = back[b].location_buffer;
if (back[b].r.statuscode == STATUSCODE_INVALID) {
hts_log_print(opt, LOG_ERROR,
"Unexpected error: %s%s not found anymore in cache",
back[b].url_adr, back[b].url_fil);
} else {
hts_log_print(opt, LOG_DEBUG, "reclaim file %s%s (%d)", back[b].url_adr,
back[b].url_fil, back[b].r.statuscode);
}
}
#endif
if (!opt->verbosedisplay) {
if (!opt->quiet) {
@@ -4438,7 +4230,6 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
}
fflush(stdout);
}
//}
// ------------------------------------------------------------
// Vérificateur d'intégrité
@@ -4606,9 +4397,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) {
lien_back delayed_back;
//char BIGSTK delayed_ctype[128];
// delayed_ctype[0] = '\0';
// strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1); // copier content-type
back_copy_static(&back[b], &delayed_back);
/* Delete entry */
@@ -4713,9 +4501,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
if (b >= 0) {
lien_back delayed_back;
//char BIGSTK delayed_ctype[128];
//delayed_ctype[0] = '\0';
//strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1); // copier content-type
back_copy_static(&back[b], &delayed_back);
/* Error */
@@ -4867,13 +4652,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
/* 'no error page' selected or file discarded by size rules! */
if (!opt->errpage || (in_error == STATUSCODE_TOO_BIG)) {
/* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
#if 0
/* No (3.43) - don't do that. We must not post-exclude an authorized link, because this will prevent the cache
system from processing it, leading to refetch it endlessly. Just accept it, and handle the error as
usual during parsing.
*/
*forbidden_url = 1; /* Forbidden! */
#endif
/* Do not post-exclude the link here (*forbidden_url): the cache system
would never process it again, and it would be refetched endlessly */
if (in_error == STATUSCODE_TOO_BIG) {
hts_log_print(opt, LOG_INFO,
"link not taken because of its size (%d bytes) at %s%s",

View File

@@ -66,7 +66,6 @@ struct htsmoduleStructExtended {
char ***filters_;
robots_wizard *robots_;
hash_struct *hash_;
//int *lien_max_;
/* Base & codebase */
char *base;

View File

@@ -265,7 +265,6 @@ T_SOC smallserver_init(int *port, char *adr) {
SOCaddr_initport(server, *port);
if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
if (listen(soc, 10) >= 0) {
// SOCaddr_inetntoa(adr, 128, server2);
strcpy(adr, h_loc);
} else {
#ifdef _WIN32
@@ -439,7 +438,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
meth = 10;
} else {
#ifdef _DEBUG
// assert(FALSE);
#endif
}
if (meth) {
@@ -1360,7 +1358,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
StringCat(headers, error_hdr);
StringCat(output, error);
//assert(file == NULL);
}
}
} else {
@@ -1388,12 +1385,10 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
!= StringLength(output)))
) {
#ifdef _DEBUG
//assert(FALSE);
#endif
}
} else {
#ifdef _DEBUG
// assert(FALSE);
#endif
}
@@ -1540,8 +1535,7 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
} while(strnotempty(test));
}
if (!strnotempty(test)) { // éviter doublons
// conv_printf(key,key);
if (!strnotempty(test)) { // éviter doublons
const size_t len = strlen(intkey);
char *const buff = (char *) malloc(len + 1);
@@ -1632,8 +1626,6 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
intkey = "";
} else {
if (loops > 0) {
//err_msg += intkey;
//err_msg += " ";
}
}
}
@@ -1740,13 +1732,7 @@ static void LANG_DELETE(void) {
}
// sélection de la langue
static void LANG_INIT(const char *path) {
//CWinApp* pApp = AfxGetApp();
//if (pApp) {
/* pApp->GetProfileInt("Language","IntId",0); */
LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
//}
}
static void LANG_INIT(const char *path) { LANG_T(path, 0); }
static int LANG_T(const char *path, int l) {
if (l >= 0) {

View File

@@ -68,16 +68,6 @@ struct find_handle_struct {
char path[2048];
};
#endif
//#ifndef HTS_DEF_FWSTRUCT_topindex_chain
//#define HTS_DEF_FWSTRUCT_topindex_chain
//typedef struct topindex_chain topindex_chain;
//#endif
//struct topindex_chain {
// int level; /* sort level */
// char *category; /* category */
// char name[2048]; /* path */
// struct topindex_chain *next; /* next element */
//};
/* Tools */
@@ -293,7 +283,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2];
char *curr;
//int n=0;
char *a;
int slash = 0;
@@ -338,7 +327,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
if (*curr == '/')
curr++;
l = link;
//c=curr;
// couper ce qui est commun
while((streql(*link, *curr)) && (*link != 0)) {
link++;
@@ -350,8 +338,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
link--;
curr--;
}
//if (*link=='/') link++;
//if (*curr=='/') curr++;
}
// calculer la profondeur du répertoire courant et remonter
@@ -362,7 +348,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
while(*a)
if (*(a++) == '/')
strlcatbuff(s, "../", ssize);
//if (strlen(s)==0) strcatbuff(s,"/");
if (slash)
strlcatbuff(s, "/", ssize); // keep it absolute!

View File

@@ -639,8 +639,7 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien
int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel)
if (back[i].status >= 0) { // signifie "lien actif"
// int ok=0; // OPTI
if (back[i].status >= 0) { // signifie "lien actif"
ok = 0;
switch (j) {
case 0: // prioritaire

View File

@@ -188,12 +188,10 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
/* Doit-on traiter les non html? */
if ((opt->getmode & HTS_GETMODE_NONHTML) == 0) { // non on ne doit pas
if (!ishtml(opt, fil)) { // non il ne faut pas
//adr[0]='\0'; // ne pas traiter ce lien, pas traiter
if (!ishtml(opt, fil)) { // non il ne faut pas
forbidden_url = 1; // interdire récupération du lien
hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr,
fil);
}
}
@@ -352,7 +350,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
} else { // adresse différente, sortir?
//if (!opt->wizard) { // mode non wizard
// doit-on traiter ce lien?.. vérifier droits de sortie
switch ((opt->travel & HTS_TRAVEL_SCOPE_MASK)) {
case HTS_TRAVEL_SAME_ADDRESS:
@@ -380,7 +377,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
if ((i > 0) && (j > 0)) {
if (!strfield2(adr + i, urladr() + j)) { // !=
if (!opt->wizard) { // mode non wizard
// printf("refused: %s\n",adr);
forbidden_url = 1; // pas même domaine
hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s",
adr, fil);
@@ -408,7 +404,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
if ((i > 0) && (j > 0)) {
if (!strfield2(adr + i, urladr() + j)) { // !-
if (!opt->wizard) { // mode non wizard
// printf("refused: %s\n",adr);
forbidden_url = 1; // pas même .xx
hts_log_print(opt, LOG_DEBUG,
"foreign location link canceled: %s%s", adr, fil);
@@ -439,7 +434,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
// récupérer les liens à côtés d'un lien (nearlink) (nvelle pos)
if (forbidden_url != 0 && opt->nearlink) {
if (!ishtml(opt, fil)) { // non html
//printf("ok %s%s\n",ad,fil);
forbidden_url = 0; // autoriser
may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner
hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil);
@@ -887,7 +881,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
}
}
}
//adr[0]='\0'; // cancel
}
// -------------------- FINAL PHASE --------------------
// Test if the "Near" test won

View File

@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com
/* ZLib */
#include "zlib.h"
//#include "zutil.h"
/* MiniZip */
#include "minizip/zip.h"

View File

@@ -54,7 +54,6 @@ static int linput(FILE * fp, char *s, int max);
#include "htswrap.h"
/* specific definitions */
//#include "htsbase.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -454,8 +453,7 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b
for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien
int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel)
if (back[i].status >= 0) { // signifie "lien actif"
// int ok=0; // OPTI
if (back[i].status >= 0) { // signifie "lien actif"
ok = 0;
switch (j) {
case 0: // prioritaire
@@ -773,28 +771,7 @@ static void sig_finish(int code) { // finir et quitter
fprintf(stderr, "\nExit requested to engine (signal %d)\n", code);
}
#ifdef _WIN32
#if 0
static void sig_ask(int code) { // demander
char s[256];
signal(code, sig_term); // quitter si encore
printf("\nQuit program/Interrupt/Cancel? (Q/I/C) ");
fflush(stdout);
scanf("%s", s);
if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
|| (s[0] == 'q') || (s[0] == 'Q'))
exit(0); // quitter
else if ((s[0] == 'i') || (s[0] == 'I')) {
if (global_opt != NULL) {
// ask for stop
global_opt->state.stop = 1;
}
}
signal(code, sig_ask); // remettre signal
}
#endif
#else
#ifndef _WIN32
static void sig_doback(int blind);
static void sig_back(int code) { // ignorer et mettre en backing
if (global_opt != NULL && !global_opt->background_on_suspend) {
@@ -809,36 +786,6 @@ static void sig_back(int code) { // ignorer et mettre en backing
}
}
#if 0
static void sig_ask(int code) { // demander
char s[256];
signal(code, sig_term); // quitter si encore
printf
("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) ");
fflush(stdout);
scanf("%s", s);
if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
|| (s[0] == 'q') || (s[0] == 'Q'))
exit(0); // quitter
else if ((s[0] == 'b') || (s[0] == 'B') || (s[0] == 'a') || (s[0] == 'A'))
sig_doback(0); // arrière plan
else if ((s[0] == 'l') || (s[0] == 'L'))
sig_doback(1); // arrière plan
else if ((s[0] == 'i') || (s[0] == 'I')) {
if (global_opt != NULL) {
// ask for stop
printf("finishing pending transfers.. please wait\n");
global_opt->state.stop = 1;
}
signal(code, sig_ask); // remettre signal
} else {
printf("cancel..\n");
signal(code, sig_ask); // remettre signal
}
}
#endif
static void sig_brpipe(int code) { // treat if necessary
signal(code, sig_brpipe);
}
@@ -951,23 +898,12 @@ static void sig_leave(int code) {
static void signal_handlers(void) {
#ifdef _WIN32
#if 0 /* BUG366763 */
signal(SIGINT, sig_ask); // ^C
#else
signal(SIGINT, sig_leave); // ^C
#endif
signal(SIGTERM, sig_finish); // kill <process>
#else
#if 0 /* BUG366763 */
signal(SIGHUP, sig_back); // close window
#endif
signal(SIGTSTP, sig_back); // ^Z
signal(SIGTERM, sig_finish); // kill <process>
#if 0 /* BUG366763 */
signal(SIGINT, sig_ask); // ^C
#else
signal(SIGINT, sig_leave); // ^C
#endif
signal(SIGPIPE, sig_brpipe); // broken pipe (write into non-opened socket)
signal(SIGCHLD, sig_ignore); // child change status
#endif

View File

@@ -250,13 +250,6 @@ static int gethost(const char *hostname, SOCaddr * server) {
struct addrinfo hints;
memset(&hints, 0, sizeof(hints));
#if 0
if (IPV6_resolver == 1) // V4 only (for bogus V6 entries)
hints.ai_family = PF_INET;
else if (IPV6_resolver == 2) // V6 only (for testing V6 only)
hints.ai_family = PF_INET6;
else
#endif
hints.ai_family = PF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
@@ -340,12 +333,7 @@ int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort,
T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM);
if (soc != INVALID_SOCKET && socICP != INVALID_SOCKET) {
//char url[HTS_URLMAXSIZE * 2];
//char method[32];
//char data[32768];
//url[0] = method[0] = data[0] = '\0';
//
printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort);
printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort);
#ifndef _WIN32
@@ -950,7 +938,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
char *command;
char *proto;
char *surl;
//int directHit = 0;
int headRequest = 0;
int listRequest = 0;
@@ -1092,10 +1079,8 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
/* Post-process request */
if (link_has_authority(surl)) {
if (strncasecmp
(surl, "http://proxytrack/",
sizeof("http://proxytrack/") - 1) == 0) {
//directHit = 1; /* Another direct hit hack */
if (strncasecmp(surl, "http://proxytrack/",
sizeof("http://proxytrack/") - 1) == 0) {
}
StringCopy(url, surl);
} else {
@@ -1116,7 +1101,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
toHit += 7;
}
/* Direct hit */
//directHit = 1;
StringCopy(url, "");
if (!link_has_authority(toHit))
StringCat(url, "http://");
@@ -1127,7 +1111,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
const char *toHit = surl + sizeof("/proxytrack/") - 1;
/* Direct hit */
//directHit = 1;
StringCopy(url, "");
if (!link_has_authority(toHit))
StringCat(url, "http://");
@@ -1611,10 +1594,7 @@ static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) {
unsigned char Opcode = buffer[0];
unsigned char Version = buffer[1];
unsigned short Message_Length = READ_NET16(&buffer[2]);
unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
//unsigned int Options = READ_NET32(&buffer[8]);
//unsigned int Option_Data = READ_NET32(&buffer[12]); /* ICP_FLAG_SRC_RTT */
//unsigned int Sender_Host_Address = READ_NET32(&buffer[16]); /* ignored */
unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
unsigned char *Payload = &buffer[20];
buffer[bufferSize] = '\0'; /* Ensure payload is NULL terminated */

View File

@@ -1035,7 +1035,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
if (unzOpenCurrentFile(index->zFile) == Z_OK) {
char headerBuff[8192 + 2];
int readSizeHeader;
//int totalHeader = 0;
int dataincache = 0;
/* For BIG comments */
@@ -1074,9 +1073,8 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified
ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag
ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved
ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition
//ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address
//ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename
ZIP_READFIELD_STRING(line, value, "Content-Disposition",
r->cdispo); // Content-disposition
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename
if (line[0] != '\0') {
int len = r->headers ? ((int) strlen(r->headers)) : 0;
@@ -1090,8 +1088,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
strcat(r->headers, "\r\n");
}
}
} while(offset < readSizeHeader && !lineEof);
//totalHeader = offset;
} while (offset < readSizeHeader && !lineEof);
/* Previous entry */
if (previous_save_[0] != '\0') {
@@ -1202,7 +1199,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
strcpy(r->msg, "Cache Read Error : Read Data");
} else
*(r->adr + r->size) = '\0';
//printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode);
} else { // erreur
r->statuscode = STATUSCODE_INVALID;
strcpy(r->msg, "Cache Memory Error");
@@ -1484,14 +1480,12 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
a += cache_brstr(a, firstline);
strcpy(cache->lastmodified, firstline);
} else {
// fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version);
fclose(cache->dat);
cache->dat = NULL;
free(use);
use = NULL;
}
} else { // non supporté
// fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline);
} else { // non supporté
fclose(cache->dat);
cache->dat = NULL;
free(use);
@@ -1500,7 +1494,6 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
/* */
} else { // Vieille version du cache
/* */
// hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format");
cache->version = 0; // cache 1.0
strcpy(cache->lastmodified, firstline);
}