rms/httrack
1
0
Fork 0
mirror of https://github.com/xroche/httrack.git synced 2026-05-17 08:26:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

Indenting cleanup for all *.[ch] files, except htsparse.c (too ugly to be automatically indented for now) and /minizip/, /mmsrip/ (external files)

Browse Source 
setup:
  indent -l80 -lc80 -nhnl -nut -bad -bap -bbo -br -brf -bli2 -brs -bls -br -ss -sai -pmt -nsaw -nsaf -nprs -i2 -ce -npsl -npcs -cs -sob -cdw -nbc -lp

logs:
indent: ./src/htsback.c:157: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:1417: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:1826: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:1833: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:1981: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:2685: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:2747: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:2861: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:3128: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsback.c:3512: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htswizard.c:140: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htswizard.c:597: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htswizard.c:598: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htswizard.c:611: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscoremain.c:97: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscoremain.c:106: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscoremain.c:106: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscoremain.c:256: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscoremain.c:262: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscoremain.c:935: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:271: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:272: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:273: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:274: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:275: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:276: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:277: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:278: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:279: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:280: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:281: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:662: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/httrack.c:752: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:413: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:414: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:415: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:416: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:417: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:418: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:419: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:420: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:421: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:422: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsweb.c:423: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:188: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:507: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:508: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:509: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:510: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:511: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:512: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:580: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.c:581: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:115: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:246: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:247: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:248: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:249: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:250: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:251: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:314: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/proxy/proxytrack.h:315: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:151: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:158: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:161: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:166: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:174: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:177: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:188: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:243: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:245: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:248: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:257: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htstools.c:647: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htsbauth.c:364: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsbauth.c:387: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htscache.c:473: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsserver.h:112: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsserver.h:187: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsjava.c:312: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsjava.c:379: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsjava.c:407: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsjava.c:472: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsjava.c:483: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htslib.c:593: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:809: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:1743: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:1874: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:1896: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:1984: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2085: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2604: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2605: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2606: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2607: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2608: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2609: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2672: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2673: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2906: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2928: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:2996: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:4802: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htslib.c:5353: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:5354: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:5355: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:5357: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:5429: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htslib.c:5435: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscore.c:208: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htscore.c:277: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htscore.c:279: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htscore.c:357: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htscore.c:394: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htscore.c:1544: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscore.c:3330: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscore.c:3361: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htshash.c:140: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htshash.c:217: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsname.c:733: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htsname.c:749: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htsname.c:933: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htsname.c:1520: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htszlib.c:76: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htszlib.c:81: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htscatchurl.c:268: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsinthash.c:156: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsinthash.c:159: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsinthash.c:176: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsinthash.c:179: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsinthash.c:226: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsinthash.c:253: Warning:old style assignment ambiguity in "=&".  Assuming "= &"
indent: ./src/htsftp.c:169: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsftp.c:177: Warning:old style assignment ambiguity in "=-".  Assuming "= -"
indent: ./src/htsfilters.c:67: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
indent: ./src/htsfilters.c:80: Warning:old style assignment ambiguity in "=*".  Assuming "= *"
This commit is contained in:
Xavier Roche
2013-05-14 20:30:28 +00:00
parent b09def438c
commit 07f4b00367
87 changed files with 22914 additions and 19189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
libtest/callbacks-example-baselinks.c
View File

@@ -25,18 +25,21 @@
#include "htsdefines.h"
/* Local function definitions */
static int process_file(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);
static int check_detectedlink(t_hts_callbackarg *carg, httrackp* opt, char* link);
static int check_detectedlink_end(t_hts_callbackarg *carg, httrackp *opt);
static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html,
int len, const char *url_address, const char *url_file);
static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt,
char *link);
static int check_detectedlink_end(t_hts_callbackarg * carg, httrackp * opt);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
@@ -45,8 +48,9 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
if (arg == NULL || *arg == '\0' || strlen(arg) >= HTS_URLMAXSIZE / 2) {
fprintf(stderr, "** callback error: arguments expected or bad arguments\n");
fprintf(stderr, "usage: httrack --wrapper modulename,base\n");
fprintf(stderr, "example: httrack --wrapper callback,http://www.example.com/\n");
return 0; /* failed */
fprintf(stderr,
"example: httrack --wrapper callback,http://www.example.com/\n");
return 0; /* failed */
} else {
char *callbacks_userdef = strdup(arg); /* userdef */
@@ -58,50 +62,56 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
fprintf(stderr, "Using root '%s'\n", callbacks_userdef);
}
return 1; /* success */
return 1; /* success */
}
static int process_file(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file) {
char* prevBase;
static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html,
int len, const char *url_address,
const char *url_file) {
char *prevBase;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, check_html)
(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
return 0; /* Abort */
}
}
/* Disable base href, if any */
if ( ( prevBase = strstr(html, "<BASE HREF=\"") ) != NULL) {
if ((prevBase = strstr(html, "<BASE HREF=\"")) != NULL) {
prevBase[1] = 'X';
}
return 1; /* success */
return 1; /* success */
}
static int check_detectedlink(t_hts_callbackarg *carg, httrackp* opt, char* link) {
const char *base = (char*) CALLBACKARG_USERDEF(carg);
static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt,
char *link) {
const char *base = (char *) CALLBACKARG_USERDEF(carg);
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, linkdetected) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, linkdetected)(CALLBACKARG_PREV_CARG(carg), opt, link)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, linkdetected)
(CALLBACKARG_PREV_CARG(carg), opt, link)) {
return 0; /* Abort */
}
}
/* The incoming (read/write) buffer is at least HTS_URLMAXSIZE bytes long */
if (strncmp(link, "http://", 7) == 0 || strncmp(link, "https://", 8) == 0) {
char temp[HTS_URLMAXSIZE * 2];
strcpy(temp, base);
strcat(temp, link);
strcpy(link, temp);
}
return 1; /* success */
return 1; /* success */
}
static int check_detectedlink_end(t_hts_callbackarg *carg, httrackp *opt) {
char *base = (char*) CALLBACKARG_USERDEF(carg);
static int check_detectedlink_end(t_hts_callbackarg * carg, httrackp * opt) {
char *base = (char *) CALLBACKARG_USERDEF(carg);
fprintf(stderr, "Unplugged ..\n");
if (base != NULL) {
@@ -111,8 +121,8 @@ static int check_detectedlink_end(t_hts_callbackarg *carg, httrackp *opt) {
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
return 1; /* success */
}

19
libtest/callbacks-example-changecontent.c
View File

@@ -25,33 +25,36 @@
#include "htsdefines.h"
/* Local function definitions */
static int postprocess(t_hts_callbackarg *carg, httrackp *opt,
char** html, int* len,
const char* url_address, const char* url_file);
static int postprocess(t_hts_callbackarg * carg, httrackp * opt, char **html,
int *len, const char *url_address, const char *url_file);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* Plug callback functions */
CHAIN_FUNCTION(opt, postprocess, postprocess, NULL);
return 1; /* success */
return 1; /* success */
}
static int postprocess(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) {
static int postprocess(t_hts_callbackarg * carg, httrackp * opt, char **html,
int *len, const char *url_address,
const char *url_file) {
char *old = *html;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, postprocess) != NULL) {
if (CALLBACKARG_PREV_FUN(carg, postprocess)(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
if (CALLBACKARG_PREV_FUN(carg, postprocess)
(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
/* Modified *html */
old = *html;
}

83
libtest/callbacks-example-contentfilter.c
View File

@@ -25,11 +25,12 @@
#include "htsdefines.h"
/* Local function definitions */
static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* address, const char* filename);
static int end(t_hts_callbackarg *carg, httrackp *opt);
static int process(t_hts_callbackarg * carg, httrackp * opt, char *html,
int len, const char *address, const char *filename);
static int end(t_hts_callbackarg * carg, httrackp * opt);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/* TOLOWER */
#define TOLOWER_(a) (a >= 'A' && a <= 'Z') ? (a + ('a' - 'A')) : a
@@ -41,32 +42,37 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
typedef struct t_my_userdef {
char stringfilter[8192];
char* stringfilters[128];
char *stringfilters[128];
} t_my_userdef;
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* Check args */
if (arg == NULL || *arg == '\0') {
fprintf(stderr, "** callback error: arguments expected or bad arguments\n");
fprintf(stderr, "usage: httrack --wrapper callback,stringtofind,stringtofind..\n");
fprintf(stderr,
"usage: httrack --wrapper callback,stringtofind,stringtofind..\n");
fprintf(stderr, "example: httrack --wrapper callback,apple,orange,lemon\n");
return 0;
} else {
t_my_userdef *userdef = (t_my_userdef*) malloc(sizeof(t_my_userdef)); /* userdef */
char * const stringfilter = userdef->stringfilter;
char** const stringfilters = userdef->stringfilters;
t_my_userdef *userdef = (t_my_userdef *) malloc(sizeof(t_my_userdef)); /* userdef */
char *const stringfilter = userdef->stringfilter;
char **const stringfilters = userdef->stringfilters;
/* */
char* a = stringfilter;
char *a = stringfilter;
int i = 0;
fprintf(stderr, "** info: wrapper_init(%s) called!\n", arg);
fprintf(stderr, "** callback example: crawling pages only if specific keywords are found\n");
fprintf(stderr,
"** callback example: crawling pages only if specific keywords are found\n");
/* stringfilters = split(arg, ','); */
strcpy(stringfilter, arg);
@@ -75,9 +81,10 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
a = strchr(a, ',');
if (a != NULL) {
*a = '\0';
a ++;
a++;
}
fprintf(stderr, "** callback info: will crawl pages with '%s' in them\n", stringfilters[i]);
fprintf(stderr, "** callback info: will crawl pages with '%s' in them\n",
stringfilters[i]);
i++;
}
stringfilters[i++] = NULL;
@@ -87,35 +94,42 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
CHAIN_FUNCTION(opt, end, end, userdef);
}
return 1; /* success */
return 1; /* success */
}
static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* address, const char* filename) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
/*char * const stringfilter = userdef->stringfilter;*/
char** const stringfilters = userdef->stringfilters;
static int process(t_hts_callbackarg * carg, httrackp * opt, char *html,
int len, const char *address, const char *filename) {
t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg);
/*char * const stringfilter = userdef->stringfilter; */
char **const stringfilters = userdef->stringfilters;
/* */
int i = 0;
int getIt = 0;
char* pos;
char *pos;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, html, len, address, filename)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, check_html)
(CALLBACKARG_PREV_CARG(carg), opt, html, len, address, filename)) {
return 0; /* Abort */
}
}
/* Process */
if (strcmp(address, "primary") == 0 && strcmp(filename, "/primary") == 0) /* primary page (list of links) */
if (strcmp(address, "primary") == 0 && strcmp(filename, "/primary") == 0) /* primary page (list of links) */
return 1;
while(stringfilters[i] != NULL && ! getIt) {
if ( ( pos = strstr(html, stringfilters[i]) ) != NULL) {
while(stringfilters[i] != NULL && !getIt) {
if ((pos = strstr(html, stringfilters[i])) != NULL) {
int j;
getIt = 1;
fprintf(stderr, "** callback info: found '%s' keyword in '%s%s', crawling this page!\n", stringfilters[i], address, filename);
fprintf(stderr,
"** callback info: found '%s' keyword in '%s%s', crawling this page!\n",
stringfilters[i], address, filename);
fprintf(stderr, "** details:\n(..)");
for(j = 0; j < 72 && pos[j] ; j++) {
for(j = 0; j < 72 && pos[j]; j++) {
if (pos[j] > 32)
fprintf(stderr, "%c", pos[j]);
else
@@ -126,15 +140,18 @@ static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len,
i++;
}
if (getIt) {
return 1; /* success */
return 1; /* success */
} else {
fprintf(stderr, "** callback info: won't parse '%s%s' (no specified keywords found)\n", address, filename);
return 0; /* this page sucks, don't parse it */
fprintf(stderr,
"** callback info: won't parse '%s%s' (no specified keywords found)\n",
address, filename);
return 0; /* this page sucks, don't parse it */
}
}
static int end(t_hts_callbackarg *carg, httrackp *opt) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
static int end(t_hts_callbackarg * carg, httrackp * opt) {
t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg);
fprintf(stderr, "** info: wrapper_exit() called!\n");
if (userdef != NULL) {
free(userdef);
@@ -143,8 +160,8 @@ static int end(t_hts_callbackarg *carg, httrackp *opt) {
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
return 1; /* success */
}

29
libtest/callbacks-example-displayheader.c
View File

@@ -25,42 +25,43 @@
#include "htsdefines.h"
/* Local function definitions */
static int process(t_hts_callbackarg *carg, httrackp *opt,
char* buff, const char* adr, const char* fil,
const char* referer_adr, const char* referer_fil,
htsblk* incoming);
static int process(t_hts_callbackarg * carg, httrackp * opt, char *buff,
const char *adr, const char *fil, const char *referer_adr,
const char *referer_fil, htsblk * incoming);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* Plug callback functions */
CHAIN_FUNCTION(opt, receivehead, process, NULL);
return 1; /* success */
return 1; /* success */
}
static int process(t_hts_callbackarg *carg, httrackp *opt,
char* buff, const char* adr, const char* fil,
const char* referer_adr, const char* referer_fil,
htsblk* incoming) {
static int process(t_hts_callbackarg * carg, httrackp * opt, char *buff,
const char *adr, const char *fil, const char *referer_adr,
const char *referer_fil, htsblk * incoming) {
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, receivehead) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, receivehead)(CALLBACKARG_PREV_CARG(carg), opt, buff, adr, fil, referer_adr, referer_fil, incoming)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, receivehead)
(CALLBACKARG_PREV_CARG(carg), opt, buff, adr, fil, referer_adr,
referer_fil, incoming)) {
return 0; /* Abort */
}
}
/* Process */
printf("[ %s%s ]\n%s\n", adr, fil, buff);
return 1; /* success */
return 1; /* success */
}

34
libtest/callbacks-example-filename.c
View File

@@ -24,10 +24,13 @@
#include "htsdefines.h"
/* Local function definitions */
static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);
static int mysavename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/* Options settings */
#include "htsopt.h"
@@ -54,33 +57,40 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* Plug callback functions */
CHAIN_FUNCTION(opt, savename, mysavename, NULL);
return 1; /* success */
return 1; /* success */
}
static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save) {
char* a;
static int mysavename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save) {
char *a;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, savename) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, savename)(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, referer_adr, referer_fil, save)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, savename)
(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete,
referer_adr, referer_fil, save)) {
return 0; /* Abort */
}
}
/* Process */
for(a = save ; *a != 0 ; a++) {
for(a = save; *a != 0; a++) {
char c = TOLOWER(*a);
if (c >= 'a' && c <= 'z')
*a = ( ( ( c - 'a' ) + 13 ) % 26 ) + 'a'; // ROT-13
*a = (((c - 'a') + 13) % 26) + 'a'; // ROT-13
}
return 1; /* success */
return 1; /* success */
}

66
libtest/callbacks-example-filename2.c
View File

@@ -21,11 +21,14 @@
#include "htsdefines.h"
/* Function definitions */
static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);
static int myend(t_hts_callbackarg *carg, httrackp *opt);
static int mysavename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save);
static int myend(t_hts_callbackarg * carg, httrackp * opt);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/* TOLOWER */
#define TOLOWER_(a) (a >= 'A' && a <= 'Z') ? (a + ('a' - 'A')) : a
@@ -45,41 +48,46 @@ typedef struct t_my_userdef {
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* Check args */
if (arg == NULL || *arg == '\0' || strchr(arg, ',') == NULL) {
fprintf(stderr, "** callback error: arguments expected or bad arguments\n");
fprintf(stderr, "usage: httrack --wrapper save-name=callback:mysavename,string1,string2\n");
fprintf(stderr, "example: httrack --wrapper save-name=callback:mysavename,foo,bar\n");
return 0; /* failed */
fprintf(stderr,
"usage: httrack --wrapper save-name=callback:mysavename,string1,string2\n");
fprintf(stderr,
"example: httrack --wrapper save-name=callback:mysavename,foo,bar\n");
return 0; /* failed */
} else {
char *pos = strchr(arg, ',');
t_my_userdef *userdef = (t_my_userdef*) malloc(sizeof(t_my_userdef));
char * const string1 = userdef->string1;
char * const string2 = userdef->string2;
t_my_userdef *userdef = (t_my_userdef *) malloc(sizeof(t_my_userdef));
char *const string1 = userdef->string1;
char *const string2 = userdef->string2;
/* Split args */
fprintf(stderr, "** info: wrapper_init(%s) called!\n", arg);
fprintf(stderr, "** callback example: changing destination filename word by another one\n");
fprintf(stderr,
"** callback example: changing destination filename word by another one\n");
string1[0] = string1[1] = '\0';
strncat(string1, arg, pos - arg);
strcpy(string2, pos + 1);
fprintf(stderr, "** callback info: will replace %s by %s in filenames!\n", string1, string2);
fprintf(stderr, "** callback info: will replace %s by %s in filenames!\n",
string1, string2);
/* Plug callback functions */
CHAIN_FUNCTION(opt, savename, mysavename, userdef);
CHAIN_FUNCTION(opt, end, myend, userdef);
}
return 1; /* success */
return 1; /* success */
}
static int myend(t_hts_callbackarg *carg, httrackp *opt) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
static int myend(t_hts_callbackarg * carg, httrackp * opt) {
t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg);
fprintf(stderr, "** info: wrapper_exit() called!\n");
if (userdef != NULL) {
@@ -89,23 +97,29 @@ static int myend(t_hts_callbackarg *carg, httrackp *opt) {
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
return 1; /* success */
}
static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
char * const string1 = userdef->string1;
char * const string2 = userdef->string2;
static int mysavename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save) {
t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg);
char *const string1 = userdef->string1;
char *const string2 = userdef->string2;
/* */
char *buff, *a, *b;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, savename) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, savename)(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, referer_adr, referer_fil, save)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, savename)
(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete,
referer_adr, referer_fil, save)) {
return 0; /* Abort */
}
}
@@ -113,9 +127,9 @@ static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_co
buff = strdup(save);
a = buff;
b = save;
*b = '\0'; /* the "save" variable points to a buffer with "sufficient" space */
*b = '\0'; /* the "save" variable points to a buffer with "sufficient" space */
while(*a) {
if (strncmp(a, string1, (int)strlen(string1)) == 0) {
if (strncmp(a, string1, (int) strlen(string1)) == 0) {
strcat(b, string2);
b += strlen(b);
a += strlen(string1);
@@ -126,5 +140,5 @@ static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_co
}
free(buff);
return 1; /* success */
return 1; /* success */
}

42
libtest/callbacks-example-filenameiisbug.c
View File

@@ -21,49 +21,61 @@
#include "htsdefines.h"
/* Function definitions */
static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);
static int mysavename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
CHAIN_FUNCTION(opt, savename, mysavename, NULL);
return 1; /* success */
return 1; /* success */
}
/*
Replaces all "offending" IIS extensions (exe, dll..) with "nice" ones
*/
static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save) {
static const char* iisBogus[] = { ".com", ".exe", ".dll", ".sh", NULL };
static const char* iisBogusReplace[] = { ".c0m", ".ex3", ".dl1", ".5h", NULL }; /* MUST be the same sizes */
char* a;
static int mysavename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save) {
static const char *iisBogus[] = { ".com", ".exe", ".dll", ".sh", NULL };
static const char *iisBogusReplace[] = { ".c0m", ".ex3", ".dl1", ".5h", NULL }; /* MUST be the same sizes */
char *a;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, savename) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, savename)(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, referer_adr, referer_fil, save)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, savename)
(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete,
referer_adr, referer_fil, save)) {
return 0; /* Abort */
}
}
/* Process */
for(a = save ; *a != '\0' ; a++) {
for(a = save; *a != '\0'; a++) {
int i;
for(i = 0 ; iisBogus[i] != NULL ; i++) {
for(i = 0; iisBogus[i] != NULL; i++) {
int j;
for(j = 0 ; iisBogus[i][j] == a[j] && iisBogus[i][j] != '\0' ; j++);
if (iisBogus[i][j] == '\0' && ( a[j] == '\0' || a[j] == '/' || a[j] == '\\' ) ) {
for(j = 0; iisBogus[i][j] == a[j] && iisBogus[i][j] != '\0'; j++) ;
if (iisBogus[i][j] == '\0'
&& (a[j] == '\0' || a[j] == '/' || a[j] == '\\')) {
strncpy(a, iisBogusReplace[i], strlen(iisBogusReplace[i]));
break;
}
}
}
return 1; /* success */
return 1; /* success */
}

74
libtest/callbacks-example-listlinks.c
View File

@@ -24,13 +24,17 @@
#include "htsdefines.h"
/* Function definitions */
static int process_file(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* url_address, const char* url_file);
static int check_detectedlink(t_hts_callbackarg *carg, httrackp *opt, char* link);
static int check_loop(t_hts_callbackarg *carg, httrackp *opt, void* back,int back_max,int back_index,int lien_tot,int lien_ntot,int stat_time,hts_stat_struct* stats);
static int end(t_hts_callbackarg *carg, httrackp *opt);
static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html,
int len, const char *url_address, const char *url_file);
static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt,
char *link);
static int check_loop(t_hts_callbackarg * carg, httrackp * opt, void *back,
int back_max, int back_index, int lien_tot, int lien_ntot,
int stat_time, hts_stat_struct * stats);
static int end(t_hts_callbackarg * carg, httrackp * opt);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
/*
This sample just lists all links in documents with the parent link:
@@ -45,15 +49,17 @@ typedef struct t_my_userdef {
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
t_my_userdef *userdef;
/* */
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* Create user-defined structure */
userdef = (t_my_userdef*) malloc(sizeof(t_my_userdef)); /* userdef */
userdef = (t_my_userdef *) malloc(sizeof(t_my_userdef)); /* userdef */
userdef->currentURLBeingParsed[0] = '\0';
/* Plug callback functions */
@@ -62,17 +68,20 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
CHAIN_FUNCTION(opt, linkdetected, check_detectedlink, userdef);
CHAIN_FUNCTION(opt, loop, check_loop, userdef);
return 1; /* success */
return 1; /* success */
}
static int process_file(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* url_address, const char* url_file) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
char * const currentURLBeingParsed = userdef->currentURLBeingParsed;
static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html,
int len, const char *url_address,
const char *url_file) {
t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg);
char *const currentURLBeingParsed = userdef->currentURLBeingParsed;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, check_html)
(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
return 0; /* Abort */
}
}
@@ -81,43 +90,50 @@ static int process_file(t_hts_callbackarg *carg, httrackp *opt, char* html, int
strcpy(currentURLBeingParsed, url_address);
strcat(currentURLBeingParsed, url_file);
return 1; /* success */
return 1; /* success */
}
static int check_detectedlink(t_hts_callbackarg *carg, httrackp *opt, char* link) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
char * const currentURLBeingParsed = userdef->currentURLBeingParsed;
static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt,
char *link) {
t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg);
char *const currentURLBeingParsed = userdef->currentURLBeingParsed;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, linkdetected) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, linkdetected)(CALLBACKARG_PREV_CARG(carg), opt, link)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, linkdetected)
(CALLBACKARG_PREV_CARG(carg), opt, link)) {
return 0; /* Abort */
}
}
/* Process */
printf("[%s] -> [%s]\n", currentURLBeingParsed, link);
return 1; /* success */
return 1; /* success */
}
static int check_loop(t_hts_callbackarg *carg, httrackp *opt, void* back,int back_max,int back_index,int lien_tot,int lien_ntot,int stat_time,hts_stat_struct* stats) {
static int fun_animation=0;
static int check_loop(t_hts_callbackarg * carg, httrackp * opt, void *back,
int back_max, int back_index, int lien_tot, int lien_ntot,
int stat_time, hts_stat_struct * stats) {
static int fun_animation = 0;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, loop) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, loop)(CALLBACKARG_PREV_CARG(carg), opt, back, back_max, back_index, lien_tot, lien_ntot, stat_time, stats)) {
return 0; /* Abort */
if (!CALLBACKARG_PREV_FUN(carg, loop)
(CALLBACKARG_PREV_CARG(carg), opt, back, back_max, back_index, lien_tot,
lien_ntot, stat_time, stats)) {
return 0; /* Abort */
}
}
/* Process */
printf("%c\r", "/-\\|"[(fun_animation++)%4]);
printf("%c\r", "/-\\|"[(fun_animation++) % 4]);
return 1;
}
static int end(t_hts_callbackarg *carg, httrackp *opt) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
static int end(t_hts_callbackarg * carg, httrackp * opt) {
t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg);
fprintf(stderr, "** info: wrapper_exit() called!\n");
if (userdef != NULL) {
free(userdef);
@@ -126,8 +142,8 @@ static int end(t_hts_callbackarg *carg, httrackp *opt) {
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
return 1; /* success */
}

50
libtest/callbacks-example-log.c
View File

@@ -24,23 +24,24 @@
#include "htsdefines.h"
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
EXTERNAL_FUNCTION int hts_unplug(httrackp * opt);
/* local function called as "check_html" callback */
static int process_file(t_hts_callbackarg *carg, httrackp *opt,
char* html, int len, const char* url_address, const char* url_file) {
void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/
static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html,
int len, const char *url_address,
const char *url_file) {
void *ourDummyArg = (void *) CALLBACKARG_USERDEF(carg); /*optional user-defined arg */
char *fmt;
(void) ourDummyArg;
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
html, len, url_address, url_file)) {
return 0; /* abort */
}
if (!CALLBACKARG_PREV_FUN(carg, check_html)
(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
return 0; /* abort */
}
}
/* log */
@@ -50,11 +51,11 @@ static int process_file(t_hts_callbackarg *carg, httrackp *opt,
hts_log(opt, "log-wrapper-info", fmt);
free(fmt);
return 1; /* success */
return 1; /* success */
}
static int start_of_mirror(t_hts_callbackarg *carg, httrackp *opt) {
const char *arginfo = (char*) CALLBACKARG_USERDEF(carg);
static int start_of_mirror(t_hts_callbackarg * carg, httrackp * opt) {
const char *arginfo = (char *) CALLBACKARG_USERDEF(carg);
fprintf(stderr, "* mirror start\n");
hts_log(opt, arginfo, "mirror started");
@@ -62,15 +63,15 @@ static int start_of_mirror(t_hts_callbackarg *carg, httrackp *opt) {
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
/* status is ok on our side, return other callabck's status */
return CALLBACKARG_PREV_FUN(carg, start)(CALLBACKARG_PREV_CARG(carg), opt);
return CALLBACKARG_PREV_FUN(carg, start) (CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
return 1; /* success */
}
/* local function called as "end" callback */
static int end_of_mirror(t_hts_callbackarg *carg, httrackp *opt) {
const char *arginfo = (char*) CALLBACKARG_USERDEF(carg);
static int end_of_mirror(t_hts_callbackarg * carg, httrackp * opt) {
const char *arginfo = (char *) CALLBACKARG_USERDEF(carg);
fprintf(stderr, "* mirror end\n");
hts_log(opt, arginfo, "mirror ended");
@@ -78,19 +79,20 @@ static int end_of_mirror(t_hts_callbackarg *carg, httrackp *opt) {
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
/* status is ok on our side, return other callabck's status */
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
return 1; /* success */
}
/*
module entry point
the function name and prototype MUST match this prototype
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
/* optional argument passed in the commandline we won't be using here */
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
@@ -98,19 +100,19 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
if (arg == NULL)
arg = "log-wrapper-info";
hts_log(opt, arg, "* plugging functions");
CHAIN_FUNCTION(opt, check_html, process_file, (char*) arg);
CHAIN_FUNCTION(opt, start, start_of_mirror, (char*) arg);
CHAIN_FUNCTION(opt, end, end_of_mirror, (char*) arg);
CHAIN_FUNCTION(opt, check_html, process_file, (char *) arg);
CHAIN_FUNCTION(opt, start, start_of_mirror, (char *) arg);
CHAIN_FUNCTION(opt, end, end_of_mirror, (char *) arg);
hts_log(opt, arg, "* module successfully plugged");
return 1; /* success */
return 1; /* success */
}
/*
module exit point
the function name and prototype MUST match this prototype
*/
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
EXTERNAL_FUNCTION int hts_unplug(httrackp * opt) {
hts_log(opt, "log-wrapper-info", "* module successfully unplugged");
return 1;
}

49
libtest/callbacks-example-simple.c
View File

@@ -24,32 +24,37 @@
#include "htsdefines.h"
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
EXTERNAL_FUNCTION int hts_unplug(httrackp * opt);
/* local function called as "check_html" callback */
static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt,
/*other parameters are callback-specific*/
char* html, int len, const char* url_address, const char* url_file) {
void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/
static int process_file(t_hts_callbackarg
/*the carg structure, holding various information */ *
carg, /*the option settings */ httrackp * opt,
/*other parameters are callback-specific */
char *html, int len, const char *url_address,
const char *url_file) {
void *ourDummyArg = (void *) CALLBACKARG_USERDEF(carg); /*optional user-defined arg */
(void) ourDummyArg;
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
html, len, url_address, url_file)) {
return 0; /* abort */
}
if (!CALLBACKARG_PREV_FUN(carg, check_html)
(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) {
return 0; /* abort */
}
}
printf("file %s%s content: %s\n", url_address, url_file, html);
return 1; /* success */
return 1; /* success */
}
/* local function called as "end" callback */
static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/
static int end_of_mirror(t_hts_callbackarg
/*the carg structure, holding various information */ *
carg, /*the option settings */ httrackp * opt) {
void *ourDummyArg = (void *) CALLBACKARG_USERDEF(carg); /*optional user-defined arg */
(void) ourDummyArg;
@@ -59,35 +64,37 @@ static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
/* status is ok on our side, return other callabck's status */
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
return 1; /* success */
}
/*
module entry point
the function name and prototype MUST match this prototype
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
/* optional argument passed in the commandline we won't be using here */
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* plug callback functions */
CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg */
NULL);
CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg */ NULL);
return 1; /* success */
return 1; /* success */
}
/*
module exit point
the function name and prototype MUST match this prototype
*/
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
EXTERNAL_FUNCTION int hts_unplug(httrackp * opt) {
fprintf(stderr, "Module unplugged");
return 1; /* success */
return 1; /* success */
}

151
libtest/example.c
View File

@@ -38,34 +38,36 @@ int main(void) {
/*
First, ask for an URL
Note: For the test, option r2 (mirror max depth=1) and --testscan (no index, no cache, do not store, no log files)
*/
char _argv[][256] = {"httrack_test" , "<URL>" , "-r3" , "--testscan" , "" };
char* argv[] = {NULL , NULL , NULL , NULL , NULL};
int argc = 0;
*/
char _argv[][256] = { "httrack_test", "<URL>", "-r3", "--testscan", "" };
char *argv[] = { NULL, NULL, NULL, NULL, NULL };
int argc = 0;
httrackp *opt;
int ret;
int ret;
while(strlen(_argv[argc])) {
argv[argc]=_argv[argc];
argv[argc] = _argv[argc];
argc++;
}
argv[argc]=NULL;
argv[argc] = NULL;
printf("HTTrackLib test program\n");
printf("Enter URL (example: www.foobar.com/index.html) :");
scanf("%s",argv[1]);
scanf("%s", argv[1]);
printf("Test: 1 depth\n");
/* Initialize the library */
/* Initialize the library */
#ifdef _WIN32
{
WORD wVersionRequested; // requested version WinSock API
WORD wVersionRequested; // requested version WinSock API
WSADATA wsadata; // Windows Sockets API data
int stat;
wVersionRequested = 0x0101;
stat = WSAStartup( wVersionRequested, &wsadata );
stat = WSAStartup(wVersionRequested, &wsadata);
if (stat != 0) {
printf("Winsock not found!\n");
return;
} else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) {
} else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) {
printf("WINSOCK.DLL does not support version 1.1\n");
WSACleanup();
return;
@@ -74,8 +76,8 @@ int main(void) {
#endif
hts_init();
/* Create option settings and set callbacks (wrappers) */
opt = hts_create_opt();
/* Create option settings and set callbacks (wrappers) */
opt = hts_create_opt();
CHAIN_FUNCTION(opt, init, httrack_wrapper_init, NULL);
CHAIN_FUNCTION(opt, uninit, httrack_wrapper_uninit, NULL);
@@ -102,15 +104,15 @@ int main(void) {
CHAIN_FUNCTION(opt, receivehead, httrack_wrapper_receiveheader, NULL);
/* Then, launch the mirror */
ret = hts_main2(argc, argv, opt);
ret = hts_main2(argc, argv, opt);
/* Wait for a key */
printf("\nPress ENTER key to exit\n");
scanf("%s",argv[1]);
scanf("%s", argv[1]);
/* Clear option state */
hts_free_opt(opt);
hts_uninit();
/* Clear option state */
hts_free_opt(opt);
hts_uninit();
#ifdef _WIN32
WSACleanup();
#endif
@@ -119,83 +121,132 @@ int main(void) {
return 0;
}
/* CALLBACK FUNCTIONS */
/* Initialize the Winsock */
static void CDECL httrack_wrapper_init(t_hts_callbackarg *carg) {
static void CDECL httrack_wrapper_init(t_hts_callbackarg * carg) {
printf("Engine started\n");
}
static void CDECL httrack_wrapper_uninit(t_hts_callbackarg *carg) {
static void CDECL httrack_wrapper_uninit(t_hts_callbackarg * carg) {
printf("Engine exited\n");
}
static int CDECL httrack_wrapper_start(t_hts_callbackarg *carg, httrackp* opt) {
static int CDECL httrack_wrapper_start(t_hts_callbackarg * carg, httrackp * opt) {
printf("Start of mirror\n");
return 1;
}
static int CDECL httrack_wrapper_chopt(t_hts_callbackarg *carg, httrackp* opt) {
return 1;
}
static int CDECL httrack_wrapper_end(t_hts_callbackarg *carg, httrackp* opt) {
static int CDECL httrack_wrapper_chopt(t_hts_callbackarg * carg, httrackp * opt) {
return 1;
}
static int CDECL httrack_wrapper_end(t_hts_callbackarg * carg, httrackp * opt) {
printf("End of mirror\n");
return 1;
}
static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file) {
printf("Parsing html file: http://%s%s\n",url_address,url_file);
return 1;
}
static int CDECL httrack_wrapper_loop(t_hts_callbackarg *carg, httrackp *opt, void* _back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats) {
static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg * carg,
httrackp * opt, char *html, int len,
const char *url_address,
const char *url_file) {
printf("Parsing html file: http://%s%s\n", url_address, url_file);
return 1;
}
static int CDECL httrack_wrapper_loop(t_hts_callbackarg * carg, httrackp * opt,
void *_back, int back_max, int back_index,
int lien_n, int lien_tot, int stat_time,
hts_stat_struct * stats) {
/* printf("..httrack_wrapper_loop called\n"); */
return 1;
}
static const char* CDECL httrack_wrapper_query(t_hts_callbackarg *carg, httrackp *opt, const char* question) {
static const char *CDECL httrack_wrapper_query(t_hts_callbackarg * carg,
httrackp * opt,
const char *question) {
/* Answer is No */
return "N";
}
static const char* CDECL httrack_wrapper_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question) {
static const char *CDECL httrack_wrapper_query2(t_hts_callbackarg * carg,
httrackp * opt,
const char *question) {
/* Answer is No */
return "N";
}
static const char* CDECL httrack_wrapper_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question) {
static const char *CDECL httrack_wrapper_query3(t_hts_callbackarg * carg,
httrackp * opt,
const char *question) {
/* Answer is "" */
return "";
}
static int CDECL httrack_wrapper_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status) {
printf("Link status tested: http://%s%s\n",adr,fil);
static int CDECL httrack_wrapper_check(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil,
int status) {
printf("Link status tested: http://%s%s\n", adr, fil);
return -1;
}
static void CDECL httrack_wrapper_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile) {
static void CDECL httrack_wrapper_pause(t_hts_callbackarg * carg,
httrackp * opt, const char *lockfile) {
/* Wait until lockfile is removed.. */
}
static void CDECL httrack_wrapper_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file) {
static void CDECL httrack_wrapper_filesave(t_hts_callbackarg * carg,
httrackp * opt, const char *file) {
}
static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link) {
printf("Link detected: %s\n",link);
static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg * carg,
httrackp * opt, char *link) {
printf("Link detected: %s\n", link);
return 1;
}
static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, void* back) {
static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg * carg,
httrackp * opt, void *back) {
return 1;
}
static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) {
static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg * carg,
httrackp * opt, char **html,
int *len,
const char *url_address,
const char *url_file) {
return 1;
}
static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) {
static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg * carg,
httrackp * opt, char **html,
int *len,
const char *url_address,
const char *url_file) {
return 1;
}
static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status) {
static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg * carg,
httrackp * opt, const char *adr,
const char *fil, const char *mime,
int status) {
return -1;
}
static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated) {
static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg * carg,
httrackp * opt, const char *adr,
const char *fil, const char *save,
int is_new, int is_modified,
int not_updated) {
}
static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag) {
static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg * carg,
httrackp * opt, char *link,
const char *start_tag) {
return 1;
}
static int CDECL httrack_wrapper_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save) {
static int CDECL httrack_wrapper_savename(t_hts_callbackarg * carg,
httrackp * opt,
const char *adr_complete,
const char *fil_complete,
const char *referer_adr,
const char *referer_fil, char *save) {
return 1;
}
static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing) {
static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg * carg,
httrackp * opt, char *buff,
const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil,
htsblk * outgoing) {
return 1;
}
static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming) {
static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg * carg,
httrackp * opt, char *buff,
const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil,
htsblk * incoming) {
return 1;
}

98
libtest/example.h
View File

@@ -9,26 +9,78 @@
#define CDECL
#endif
static void CDECL httrack_wrapper_init(t_hts_callbackarg *carg);
static void CDECL httrack_wrapper_uninit(t_hts_callbackarg *carg);
static int CDECL httrack_wrapper_start(t_hts_callbackarg *carg, httrackp* opt);
static int CDECL httrack_wrapper_chopt(t_hts_callbackarg *carg, httrackp* opt);
static int CDECL httrack_wrapper_end(t_hts_callbackarg *carg, httrackp* opt);
static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file);
static int CDECL httrack_wrapper_loop(t_hts_callbackarg *carg, httrackp *opt, void* _back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats);
static const char* CDECL httrack_wrapper_query(t_hts_callbackarg *carg, httrackp *opt, const char* question);
static const char* CDECL httrack_wrapper_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question);
static const char* CDECL httrack_wrapper_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question);
static int CDECL httrack_wrapper_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status);
static void CDECL httrack_wrapper_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile);
static void CDECL httrack_wrapper_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file);
static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link);
static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, void* back);
static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file);
static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file);
static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status);
static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated);
static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag);
static int CDECL httrack_wrapper_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save);
static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);
static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);
static void CDECL httrack_wrapper_init(t_hts_callbackarg * carg);
static void CDECL httrack_wrapper_uninit(t_hts_callbackarg * carg);
static int CDECL httrack_wrapper_start(t_hts_callbackarg * carg,
httrackp * opt);
static int CDECL httrack_wrapper_chopt(t_hts_callbackarg * carg,
httrackp * opt);
static int CDECL httrack_wrapper_end(t_hts_callbackarg * carg, httrackp * opt);
static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg * carg,
httrackp * opt, char *html, int len,
const char *url_address,
const char *url_file);
static int CDECL httrack_wrapper_loop(t_hts_callbackarg * carg, httrackp * opt,
void *_back, int back_max, int back_index,
int lien_n, int lien_tot, int stat_time,
hts_stat_struct * stats);
static const char *CDECL httrack_wrapper_query(t_hts_callbackarg * carg,
httrackp * opt,
const char *question);
static const char *CDECL httrack_wrapper_query2(t_hts_callbackarg * carg,
httrackp * opt,
const char *question);
static const char *CDECL httrack_wrapper_query3(t_hts_callbackarg * carg,
httrackp * opt,
const char *question);
static int CDECL httrack_wrapper_check(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil,
int status);
static void CDECL httrack_wrapper_pause(t_hts_callbackarg * carg,
httrackp * opt, const char *lockfile);
static void CDECL httrack_wrapper_filesave(t_hts_callbackarg * carg,
httrackp * opt, const char *file);
static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg * carg,
httrackp * opt, char *link);
static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg * carg,
httrackp * opt, void *back);
static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg * carg,
httrackp * opt, char **html,
int *len,
const char *url_address,
const char *url_file);
static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg * carg,
httrackp * opt, char **html,
int *len,
const char *url_address,
const char *url_file);
static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg * carg,
httrackp * opt, const char *adr,
const char *fil, const char *mime,
int status);
static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg * carg,
httrackp * opt, const char *adr,
const char *fil, const char *save,
int is_new, int is_modified,
int not_updated);
static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg * carg,
httrackp * opt, char *link,
const char *start_tag);
static int CDECL httrack_wrapper_savename(t_hts_callbackarg * carg,
httrackp * opt,
const char *adr_complete,
const char *fil_complete,
const char *referer_adr,
const char *referer_fil, char *save);
static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg * carg,
httrackp * opt, char *buff,
const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil,
htsblk * outgoing);
static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg * carg,
httrackp * opt, char *buff,
const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil,
htsblk * incoming);

5
src/hts-indextmpl.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Index.html templates file */
/* Author: Xavier Roche */
@@ -37,7 +34,6 @@ Please visit our Website: http://www.httrack.com
#ifndef HTTRACK_DEFTMPL
#define HTTRACK_DEFTMPL
/* Index for each project */
/*
regen:
@@ -350,7 +346,6 @@ regen:
""LF\
""LF
/* Other files (fade and backblue images) */
#define HTS_LOG_SECURITY_WARNING "note:\tthe hts-log.txt file, and hts-cache folder, may contain sensitive information,"LF\

665
src/htsalias.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsalias.c subroutines: */
/* alias for command-line options and config files */
@@ -42,8 +39,8 @@ Please visit our Website: http://www.httrack.com
#include "htsalias.h"
#include "htsglobal.h"
void linput(FILE* fp,char* s,int max);
void hts_lowcase(char* s);
void linput(FILE * fp, char *s, int max);
void hts_lowcase(char *s);
#define _NOT_NULL(a) ( (a!=NULL) ? (a) : "" )
// #define is_realspace(c) (strchr(" \x0d\x0a\x09\x0b\x0c",(c))!=NULL)
@@ -70,7 +67,6 @@ void hts_lowcase(char* s);
argc++
// END OF COPY OF cmdl_ins in htsmain.c
/*
Aliases for command-line and config file definitions
These definitions can be used:
@@ -90,174 +86,193 @@ void hts_lowcase(char* s);
param1 : this option must be alone, and needs one distinct parameter (-P <path>)
param0 : this option must be alone, but the parameter should be put together (+*.gif)
*/
const char* hts_optalias[][4] = {
const char *hts_optalias[][4] = {
/* {"","","",""}, */
{"path","-O","param1","output path"},
{"chroot","-%O","param1","default top path"},
{"mirror","-w","single",""},
{"mirror-wizard","-W","single",""},
{"get-files","-g","single",""},
{"quiet","-q","single",""},
{"mirrorlinks","-Y","single",""},
{"proxy","-P","param1","proxy name:port"},
{"bind","-%b","param1","hostname to bind"},
{"httpproxy-ftp","-%f","param",""},
{"depth","-r","param",""},{"recurse-levels","-r","param",""},
{"ext-depth","-%e","param",""},
{"max-files","-m","param",""},
{"max-size","-M","param",""},
{"max-time","-E","param",""},
{"max-mms-time","-%m","param",""},
{"max-rate","-A","param",""},
{"max-pause","-G","param",""},
{"sockets","-c","param","number of simultaneous connections allowed"},{"socket","-c","param","number of simultaneous connections allowed"},{"connection","-c","param","number of simultaneous connections allowed"},
{"connection-per-second","-%c","param","number of connection per second allowed"},
{"timeout","-T","",""},
{"retries","-R","param","number of retries for non-fatal errors"},
{"min-rate","-J","param",""},
{"host-control","-H","param",""},
{"extended-parsing","-%P","param",""},
{"near","-n","single",""},
{"delayed-type-check","-%N","single",""},
{"cached-delayed-type-check","-%D","single",""},
{"delayed-type-check-always","-%N2","single",""},
{"disable-security-limits","-%!","single",""},
{"test","-t","single",""},
{"list","-%L","param1",""},
{"urllist","-%S","param1",""},
{"language","-%l","param1",""}, {"lang","-%l","param1",""},
{"structure","-N","param",""}, {"user-structure","-N","param1",""},
{"long-names","-L","param",""},
{"keep-links","-K","param",""},
{"mime-html","-%M","single",""}, {"mht","-%M","single",""},
{"replace-external","-x","single",""},
{"disable-passwords","-%x","single",""},{"disable-password","-%x","single",""},
{"include-query-string","-%q","single",""},
{"generate-errors","-o","single",""},
{"do-not-generate-errors","-o0","single",""},
{"purge-old","-X","param",""},
{"cookies","-b","param",""},
{"check-type","-u","param",""},
{"assume","-%A","param1",""}, {"mimetype","-%A","param1",""},
{"parse-java","-j","param",""},
{"protocol","-@i","param",""},
{"robots","-s","param",""},
{"http-10","-%h","single",""},{"http-1.0","-%h","single",""},
{"keep-alive","-%k","single",""},
{"build-top-index","-%i","single",""},
{"disable-compression","-%z","single",""},
{"tolerant","-%B","single",""},
{"updatehack","-%s","single",""}, {"sizehack","-%s","single",""},
{"urlhack","-%u","single",""},
{"user-agent","-F","param1","user-agent identity"},
{"referer","-%R","param1","default referer URL"},
{"from","-%E","param1","from email address"},
{"footer","-%F","param1",""},
{"cache","-C","param","number of retries for non-fatal errors"},
{"store-all-in-cache","-k","single",""},
{"do-not-recatch","-%n","single",""},
{"do-not-log","-Q","single",""},
{"extra-log","-z","single",""},
{"debug-log","-Z","single",""},
{"verbose","-v","single",""},
{"file-log","-f","single",""},
{"single-log","-f2","single",""},
{"index","-I","single",""},
{"search-index","-%I","single",""},
{"priority","-p","param",""},
{"debug-headers","-%H","single",""},
{"userdef-cmd","-V","param1",""},
{"callback","-%W","param1","plug an external callback"}, {"wrapper","-%W","param1","plug an external callback"},
{"structure","-N","param1","user-defined structure"},
{"usercommand","-V","param1","user-defined command"},
{"display","-%v","single","show files transfered and other funny realtime information"},
{"dos83","-L0","single",""},
{"iso9660","-L2","single",""},
{"disable-module","-%w","param1",""},
{"no-background-on-suspend","-y0","single",""},
{"background-on-suspend","-y","single",""},
{"utf8-conversion","-%T","single",""},
{"no-utf8-conversion","-%T0","single",""},
{"path", "-O", "param1", "output path"},
{"chroot", "-%O", "param1", "default top path"},
{"mirror", "-w", "single", ""},
{"mirror-wizard", "-W", "single", ""},
{"get-files", "-g", "single", ""},
{"quiet", "-q", "single", ""},
{"mirrorlinks", "-Y", "single", ""},
{"proxy", "-P", "param1", "proxy name:port"},
{"bind", "-%b", "param1", "hostname to bind"},
{"httpproxy-ftp", "-%f", "param", ""},
{"depth", "-r", "param", ""}, {"recurse-levels", "-r", "param", ""},
{"ext-depth", "-%e", "param", ""},
{"max-files", "-m", "param", ""},
{"max-size", "-M", "param", ""},
{"max-time", "-E", "param", ""},
{"max-mms-time", "-%m", "param", ""},
{"max-rate", "-A", "param", ""},
{"max-pause", "-G", "param", ""},
{"sockets", "-c", "param", "number of simultaneous connections allowed"},
{"socket", "-c", "param", "number of simultaneous connections allowed"},
{"connection", "-c", "param", "number of simultaneous connections allowed"},
{"connection-per-second", "-%c", "param",
"number of connection per second allowed"},
{"timeout", "-T", "", ""},
{"retries", "-R", "param", "number of retries for non-fatal errors"},
{"min-rate", "-J", "param", ""},
{"host-control", "-H", "param", ""},
{"extended-parsing", "-%P", "param", ""},
{"near", "-n", "single", ""},
{"delayed-type-check", "-%N", "single", ""},
{"cached-delayed-type-check", "-%D", "single", ""},
{"delayed-type-check-always", "-%N2", "single", ""},
{"disable-security-limits", "-%!", "single", ""},
{"test", "-t", "single", ""},
{"list", "-%L", "param1", ""},
{"urllist", "-%S", "param1", ""},
{"language", "-%l", "param1", ""}, {"lang", "-%l", "param1", ""},
{"structure", "-N", "param", ""}, {"user-structure", "-N", "param1", ""},
{"long-names", "-L", "param", ""},
{"keep-links", "-K", "param", ""},
{"mime-html", "-%M", "single", ""}, {"mht", "-%M", "single", ""},
{"replace-external", "-x", "single", ""},
{"disable-passwords", "-%x", "single", ""}, {"disable-password", "-%x",
"single", ""},
{"include-query-string", "-%q", "single", ""},
{"generate-errors", "-o", "single", ""},
{"do-not-generate-errors", "-o0", "single", ""},
{"purge-old", "-X", "param", ""},
{"cookies", "-b", "param", ""},
{"check-type", "-u", "param", ""},
{"assume", "-%A", "param1", ""}, {"mimetype", "-%A", "param1", ""},
{"parse-java", "-j", "param", ""},
{"protocol", "-@i", "param", ""},
{"robots", "-s", "param", ""},
{"http-10", "-%h", "single", ""}, {"http-1.0", "-%h", "single", ""},
{"keep-alive", "-%k", "single", ""},
{"build-top-index", "-%i", "single", ""},
{"disable-compression", "-%z", "single", ""},
{"tolerant", "-%B", "single", ""},
{"updatehack", "-%s", "single", ""}, {"sizehack", "-%s", "single", ""},
{"urlhack", "-%u", "single", ""},
{"user-agent", "-F", "param1", "user-agent identity"},
{"referer", "-%R", "param1", "default referer URL"},
{"from", "-%E", "param1", "from email address"},
{"footer", "-%F", "param1", ""},
{"cache", "-C", "param", "number of retries for non-fatal errors"},
{"store-all-in-cache", "-k", "single", ""},
{"do-not-recatch", "-%n", "single", ""},
{"do-not-log", "-Q", "single", ""},
{"extra-log", "-z", "single", ""},
{"debug-log", "-Z", "single", ""},
{"verbose", "-v", "single", ""},
{"file-log", "-f", "single", ""},
{"single-log", "-f2", "single", ""},
{"index", "-I", "single", ""},
{"search-index", "-%I", "single", ""},
{"priority", "-p", "param", ""},
{"debug-headers", "-%H", "single", ""},
{"userdef-cmd", "-V", "param1", ""},
{"callback", "-%W", "param1", "plug an external callback"}, {"wrapper", "-%W",
"param1",
"plug an external callback"},
{"structure", "-N", "param1", "user-defined structure"},
{"usercommand", "-V", "param1", "user-defined command"},
{"display", "-%v", "single",
"show files transfered and other funny realtime information"},
{"dos83", "-L0", "single", ""},
{"iso9660", "-L2", "single", ""},
{"disable-module", "-%w", "param1", ""},
{"no-background-on-suspend", "-y0", "single", ""},
{"background-on-suspend", "-y", "single", ""},
{"utf8-conversion", "-%T", "single", ""},
{"no-utf8-conversion", "-%T0", "single", ""},
/* */
/* DEPRECATED */
{"stay-on-same-dir","-S","single","stay on the same directory - DEPRECATED"},
{"can-go-down","-D","single","can only go down into subdirs - DEPRECATED"},
{"can-go-up","-U","single","can only go to upper directories- DEPRECATED"},
{"can-go-up-and-down","-B","single","can both go up&down into the directory structure - DEPRECATED"},
{"stay-on-same-address","-a","single","stay on the same address - DEPRECATED"},
{"stay-on-same-domain","-d","single","stay on the same principal domain - DEPRECATED"},
{"stay-on-same-tld","-l","single","stay on the same TLD (eg: .com) - DEPRECATED"},
{"go-everywhere","-e","single","go everywhere on the web - DEPRECATED"},
{"stay-on-same-dir", "-S", "single",
"stay on the same directory - DEPRECATED"},
{"can-go-down", "-D", "single", "can only go down into subdirs - DEPRECATED"},
{"can-go-up", "-U", "single", "can only go to upper directories- DEPRECATED"},
{"can-go-up-and-down", "-B", "single",
"can both go up&down into the directory structure - DEPRECATED"},
{"stay-on-same-address", "-a", "single",
"stay on the same address - DEPRECATED"},
{"stay-on-same-domain", "-d", "single",
"stay on the same principal domain - DEPRECATED"},
{"stay-on-same-tld", "-l", "single",
"stay on the same TLD (eg: .com) - DEPRECATED"},
{"go-everywhere", "-e", "single", "go everywhere on the web - DEPRECATED"},
/* Badly documented */
{"debug-testfilters","-#0","param1","debug: test filters"},
{"advanced-flushlogs","-#f","single",""},
{"advanced-maxfilters","-#F","param",""},
{"version","-#h","single",""},
{"debug-scanstdin","-#K","single",""},
{"advanced-maxlinks","-#L","single",""},
{"advanced-progressinfo","-#p","single","deprecated"},
{"catch-url","-#P","single","catch complex URL through proxy"},
/*{"debug-oldftp","-#R","single",""},*/
{"debug-xfrstats","-#T","single",""},
{"advanced-wait","-#u","single",""},
{"debug-ratestats","-#Z","single",""},
{"exec","-#!","param1",""},
{"fast-engine","-#X","single","Enable fast routines"},
{"debug-overflows","-#X0","single","Attempt to detect buffer overflows"},
{"debug-cache","-#C","param1","List files in the cache"},
{"extract-cache","-#C","single","Extract meta-data"},
{"debug-parsing","-#d","single","debug: test parser"},
{"repair-cache","-#R","single","repair the damaged cache ZIP file"}, {"repair","-#R","single",""},
{"debug-testfilters", "-#0", "param1", "debug: test filters"},
{"advanced-flushlogs", "-#f", "single", ""},
{"advanced-maxfilters", "-#F", "param", ""},
{"version", "-#h", "single", ""},
{"debug-scanstdin", "-#K", "single", ""},
{"advanced-maxlinks", "-#L", "single", ""},
{"advanced-progressinfo", "-#p", "single", "deprecated"},
{"catch-url", "-#P", "single", "catch complex URL through proxy"},
/*{"debug-oldftp","-#R","single",""}, */
{"debug-xfrstats", "-#T", "single", ""},
{"advanced-wait", "-#u", "single", ""},
{"debug-ratestats", "-#Z", "single", ""},
{"exec", "-#!", "param1", ""},
{"fast-engine", "-#X", "single", "Enable fast routines"},
{"debug-overflows", "-#X0", "single", "Attempt to detect buffer overflows"},
{"debug-cache", "-#C", "param1", "List files in the cache"},
{"extract-cache", "-#C", "single", "Extract meta-data"},
{"debug-parsing", "-#d", "single", "debug: test parser"},
{"repair-cache", "-#R", "single", "repair the damaged cache ZIP file"},
{"repair", "-#R", "single", ""},
/* STANDARD ALIASES */
{"spider","-p0C0I0t","single",""},
{"testsite","-p0C0I0t","single",""},
{"testlinks","-r1p0C0I0t","single",""}, {"test","-r1p0C0I0t","single",""}, {"bookmark","-r1p0C0I0t","single",""},
{"mirror","-w","single",""},
{"testscan","-p0C0I0Q","single",""}, {"scan","-p0C0I0Q","single",""}, {"check","-p0C0I0Q","single",""},
{"skeleton","-p1","single",""},
{"preserve","-%p","single",""},
{"get","-qg","single",""},
{"update","-iC2","single",""},
{"continue","-iC1","single",""}, {"restart","-iC1","single",""},
{"continue","-i","single",""}, /* for help alias */
{"sucker","-r999","single",""},
{"help","-h","single",""}, {"documentation","-h","single",""}, {"doc","-h","single",""},
{"wide","-c32","single",""},
{"tiny","-c1","single",""},
{"ultrawide","-c48","single",""},
{"http10","-%h","single",""},
{"filelist","-%L","single",""}, {"list","-%L","single",""},
{"filterlist","-%S","single",""},
{"spider", "-p0C0I0t", "single", ""},
{"testsite", "-p0C0I0t", "single", ""},
{"testlinks", "-r1p0C0I0t", "single", ""}, {"test", "-r1p0C0I0t", "single",
""}, {"bookmark", "-r1p0C0I0t",
"single", ""},
{"mirror", "-w", "single", ""},
{"testscan", "-p0C0I0Q", "single", ""}, {"scan", "-p0C0I0Q", "single", ""},
{"check", "-p0C0I0Q", "single", ""},
{"skeleton", "-p1", "single", ""},
{"preserve", "-%p", "single", ""},
{"get", "-qg", "single", ""},
{"update", "-iC2", "single", ""},
{"continue", "-iC1", "single", ""}, {"restart", "-iC1", "single", ""},
{"continue", "-i", "single", ""}, /* for help alias */
{"sucker", "-r999", "single", ""},
{"help", "-h", "single", ""}, {"documentation", "-h", "single", ""}, {"doc",
"-h",
"single",
""},
{"wide", "-c32", "single", ""},
{"tiny", "-c1", "single", ""},
{"ultrawide", "-c48", "single", ""},
{"http10", "-%h", "single", ""},
{"filelist", "-%L", "single", ""}, {"list", "-%L", "single", ""},
{"filterlist", "-%S", "single", ""},
/* END OF ALIASES */
/* Filters */
{"allow","+","param0","allow filter"},
{"deny","-","param0","deny filter"},
{"allow", "+", "param0", "allow filter"},
{"deny", "-", "param0", "deny filter"},
/* */
/* URLs */
{"add","","param0","add URLs"},
{"add", "", "param0", "add URLs"},
/* */
/* Specific */
{"user","-%U","param1","output path"},
{"user", "-%U", "param1", "output path"},
/* */
/* Internal */
{"catchurl","--catchurl","single","catch complex URL through proxy"},
{"updatehttrack","--updatehttrack","single","update HTTrack Website Copier"},
{"clean","--clean","single","clean up log files and cache"},
{"tide","--clean","single","clean up log files and cache"},
{"autotest","-#T","single",""},
{"catchurl", "--catchurl", "single", "catch complex URL through proxy"},
{"updatehttrack", "--updatehttrack", "single",
"update HTTrack Website Copier"},
{"clean", "--clean", "single", "clean up log files and cache"},
{"tide", "--clean", "single", "clean up log files and cache"},
{"autotest", "-#T", "single", ""},
/* */
{"","","",""}
{"", "", "", ""}
};
/*
Check for alias in command-line
argc,argv as in main()
@@ -267,139 +282,146 @@ const char* hts_optalias[][4] = {
return value: number of arguments treated (0 if error)
*/
int optalias_check(int argc,const char * const * argv,int n_arg,
int* return_argc,char** return_argv,
char* return_error) {
return_error[0]='\0';
*return_argc=1;
if (argv[n_arg][0]=='-')
if (argv[n_arg][1]=='-') {
char command[1000];
char param[1000];
char addcommand[256];
/* */
char* position;
int need_param=1;
//int return_param=0;
int pos;
command[0]=param[0]=addcommand[0]='\0';
int optalias_check(int argc, const char *const *argv, int n_arg,
int *return_argc, char **return_argv, char *return_error) {
return_error[0] = '\0';
*return_argc = 1;
if (argv[n_arg][0] == '-')
if (argv[n_arg][1] == '-') {
char command[1000];
char param[1000];
char addcommand[256];
/* --sockets=8 */
if ( (position=strchr(argv[n_arg],'=')) ) {
/* Copy command */
strncatbuff(command,argv[n_arg]+2,(int) (position - (argv[n_arg]+2)) );
/* Copy parameter */
strcpybuff(param,position+1);
}
/* --nocache */
else if (strncmp(argv[n_arg]+2,"no",2)==0) {
strcpybuff(command,argv[n_arg]+4);
strcpybuff(param,"0");
}
/* --sockets 8 */
else {
if (strncmp(argv[n_arg]+2,"wide-",5)==0) {
strcpybuff(addcommand,"c32");
strcpybuff(command,strchr(argv[n_arg]+2,'-')+1);
} else if (strncmp(argv[n_arg]+2,"tiny-",5)==0) {
strcpybuff(addcommand,"c1");
strcpybuff(command,strchr(argv[n_arg]+2,'-')+1);
} else
strcpybuff(command,argv[n_arg]+2);
need_param=2;
}
/* */
char *position;
int need_param = 1;
/* Now solve the alias */
pos=optalias_find(command);
if (pos>=0) {
/* Copy real name */
strcpybuff(command,hts_optalias[pos][1]);
/* With parameters? */
if (strncmp(hts_optalias[pos][2],"param",5)==0) {
/* Copy parameters? */
if (need_param == 2) {
if ((n_arg+1>=argc) || (argv[n_arg+1][0]=='-')) { /* no supplemental parameter */
sprintf(return_error,
"Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n",
command,command,_NOT_NULL(optalias_help(command)));
return 0;
}
strcpybuff(param,argv[n_arg+1]);
need_param=2;
}
} else
need_param=1;
//int return_param=0;
int pos;
/* Final result */
command[0] = param[0] = addcommand[0] = '\0';
/* Must be alone (-P /tmp) */
if (strcmp(hts_optalias[pos][2],"param1")==0) {
strcpybuff(return_argv[0],command);
strcpybuff(return_argv[1],param);
*return_argc=2; /* 2 parameters returned */
}
/* Alone with parameter (+*.gif) */
else if (strcmp(hts_optalias[pos][2],"param0")==0) {
/* Command */
strcpybuff(return_argv[0],command);
strcatbuff(return_argv[0],param);
/* --sockets=8 */
if ((position = strchr(argv[n_arg], '='))) {
/* Copy command */
strncatbuff(command, argv[n_arg] + 2,
(int) (position - (argv[n_arg] + 2)));
/* Copy parameter */
strcpybuff(param, position + 1);
}
/* Together (-c8) */
/* --nocache */
else if (strncmp(argv[n_arg] + 2, "no", 2) == 0) {
strcpybuff(command, argv[n_arg] + 4);
strcpybuff(param, "0");
}
/* --sockets 8 */
else {
/* Command */
strcpybuff(return_argv[0],command);
/* Parameters accepted */
if (strncmp(hts_optalias[pos][2],"param",5)==0) {
/* --cache=off or --index=on */
if (strcmp(param,"off")==0)
strcatbuff(return_argv[0],"0");
else if (strcmp(param,"on")==0) {
// on is the default
// strcatbuff(return_argv[0],"1");
} else
strcatbuff(return_argv[0],param);
}
*return_argc=1; /* 1 parameter returned */
if (strncmp(argv[n_arg] + 2, "wide-", 5) == 0) {
strcpybuff(addcommand, "c32");
strcpybuff(command, strchr(argv[n_arg] + 2, '-') + 1);
} else if (strncmp(argv[n_arg] + 2, "tiny-", 5) == 0) {
strcpybuff(addcommand, "c1");
strcpybuff(command, strchr(argv[n_arg] + 2, '-') + 1);
} else
strcpybuff(command, argv[n_arg] + 2);
need_param = 2;
}
} else {
sprintf(return_error,"Unknown option: %s\n",command);
return 0;
/* Now solve the alias */
pos = optalias_find(command);
if (pos >= 0) {
/* Copy real name */
strcpybuff(command, hts_optalias[pos][1]);
/* With parameters? */
if (strncmp(hts_optalias[pos][2], "param", 5) == 0) {
/* Copy parameters? */
if (need_param == 2) {
if ((n_arg + 1 >= argc) || (argv[n_arg + 1][0] == '-')) { /* no supplemental parameter */
sprintf(return_error,
"Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n",
command, command, _NOT_NULL(optalias_help(command)));
return 0;
}
strcpybuff(param, argv[n_arg + 1]);
need_param = 2;
}
} else
need_param = 1;
/* Final result */
/* Must be alone (-P /tmp) */
if (strcmp(hts_optalias[pos][2], "param1") == 0) {
strcpybuff(return_argv[0], command);
strcpybuff(return_argv[1], param);
*return_argc = 2; /* 2 parameters returned */
}
/* Alone with parameter (+*.gif) */
else if (strcmp(hts_optalias[pos][2], "param0") == 0) {
/* Command */
strcpybuff(return_argv[0], command);
strcatbuff(return_argv[0], param);
}
/* Together (-c8) */
else {
/* Command */
strcpybuff(return_argv[0], command);
/* Parameters accepted */
if (strncmp(hts_optalias[pos][2], "param", 5) == 0) {
/* --cache=off or --index=on */
if (strcmp(param, "off") == 0)
strcatbuff(return_argv[0], "0");
else if (strcmp(param, "on") == 0) {
// on is the default
// strcatbuff(return_argv[0],"1");
} else
strcatbuff(return_argv[0], param);
}
*return_argc = 1; /* 1 parameter returned */
}
} else {
sprintf(return_error, "Unknown option: %s\n", command);
return 0;
}
return need_param;
}
return need_param;
}
/* Check -O <path> */
{
int pos;
if ((pos=optreal_find(argv[n_arg]))>=0) {
if ( (strcmp(hts_optalias[pos][2],"param1")==0) || (strcmp(hts_optalias[pos][2],"param0")==0)) {
if ((n_arg+1>=argc) || (argv[n_arg+1][0]=='-')) { /* no supplemental parameter */
if ((pos = optreal_find(argv[n_arg])) >= 0) {
if ((strcmp(hts_optalias[pos][2], "param1") == 0)
|| (strcmp(hts_optalias[pos][2], "param0") == 0)) {
if ((n_arg + 1 >= argc) || (argv[n_arg + 1][0] == '-')) { /* no supplemental parameter */
sprintf(return_error,
"Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n",
argv[n_arg],argv[n_arg],_NOT_NULL(optalias_help(argv[n_arg])));
"Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n",
argv[n_arg], argv[n_arg],
_NOT_NULL(optalias_help(argv[n_arg])));
return 0;
}
/* Copy parameters */
strcpybuff(return_argv[0],argv[n_arg]);
strcpybuff(return_argv[1],argv[n_arg+1]);
strcpybuff(return_argv[0], argv[n_arg]);
strcpybuff(return_argv[1], argv[n_arg + 1]);
/* And return */
*return_argc=2; /* 2 parameters returned */
return 2; /* 2 parameters used */
*return_argc = 2; /* 2 parameters returned */
return 2; /* 2 parameters used */
}
}
}
/* Copy and return other unknown option */
strcpybuff(return_argv[0],argv[n_arg]);
strcpybuff(return_argv[0], argv[n_arg]);
return 1;
}
/* Finds the <token> option alias and returns the index, or -1 if failed */
int optalias_find(const char* token) {
int optalias_find(const char *token) {
if (token[0] != '\0') {
int i=0;
int i = 0;
while(hts_optalias[i][0][0] != '\0') {
if (strcmp(token,hts_optalias[i][0])==0) {
if (strcmp(token, hts_optalias[i][0]) == 0) {
return i;
}
i++;
@@ -409,11 +431,12 @@ int optalias_find(const char* token) {
}
/* Finds the <token> real option and returns the index, or -1 if failed */
int optreal_find(const char* token) {
int optreal_find(const char *token) {
if (token[0] != '\0') {
int i=0;
int i = 0;
while(hts_optalias[i][0][0] != '\0') {
if (strcmp(token,hts_optalias[i][1])==0) {
if (strcmp(token, hts_optalias[i][1]) == 0) {
return i;
}
i++;
@@ -422,23 +445,24 @@ int optreal_find(const char* token) {
return -1;
}
const char* optreal_value(int p) {
const char *optreal_value(int p) {
return hts_optalias[p][1];
}
const char* optalias_value(int p) {
const char *optalias_value(int p) {
return hts_optalias[p][0];
}
const char* opttype_value(int p) {
const char *opttype_value(int p) {
return hts_optalias[p][2];
}
const char* opthelp_value(int p) {
const char *opthelp_value(int p) {
return hts_optalias[p][3];
}
/* Help for option <token>, empty if not available, or NULL if unknown <token> */
const char* optalias_help(const char* token) {
int pos=optalias_find(token);
if (pos>=0)
const char *optalias_help(const char *token) {
int pos = optalias_find(token);
if (pos >= 0)
return hts_optalias[pos][3];
else
return NULL;
@@ -452,76 +476,89 @@ const char* optalias_help(const char* token) {
deny ad.*
*/
/* Note: NOT utf-8 */
int optinclude_file(const char* name,
int* argc,char** argv,char* x_argvblk,int* x_ptr) {
FILE* fp;
fp=fopen(name,"rb");
int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
int *x_ptr) {
FILE *fp;
fp = fopen(name, "rb");
if (fp) {
char line[256];
int insert_after=1; /* first, insert after program filename */
int insert_after = 1; /* first, insert after program filename */
while(!feof(fp)) {
char *a,*b;
char *a, *b;
int result;
/* read line */
linput(fp,line,250);
linput(fp, line, 250);
hts_lowcase(line);
if (strnotempty(line)) {
/* no comment line: # // ; */
if (strchr("#/;",line[0])==NULL) {
if (strchr("#/;", line[0]) == NULL) {
/* right trim */
a=line+strlen(line)-1;
while(is_realspace(*a)) *(a--) = '\0';
a = line + strlen(line) - 1;
while(is_realspace(*a))
*(a--) = '\0';
/* jump "set " and spaces */
a=line;
while(is_realspace(*a)) a++;
if (strncmp(a,"set",3)==0) {
if (is_realspace(*(a+3))) {
a+=4;
a = line;
while(is_realspace(*a))
a++;
if (strncmp(a, "set", 3) == 0) {
if (is_realspace(*(a + 3))) {
a += 4;
}
}
while(is_realspace(*a)) a++;
while(is_realspace(*a))
a++;
/* delete = ("sockets=8") */
if ( (b=strchr(a,'=')) )
*b=' ';
if ((b = strchr(a, '=')))
*b = ' ';
/* isolate option and parameter */
b=a;
while( (!is_realspace(*b)) && (*b) ) b++;
b = a;
while((!is_realspace(*b)) && (*b))
b++;
if (*b) {
*b='\0';
*b = '\0';
b++;
}
/* a is now the option, b the parameter */
{
int return_argc;
char return_error[256];
char _tmp_argv[4][HTS_CDLMAXSIZE];
char* tmp_argv[4];
tmp_argv[0]=_tmp_argv[0]; tmp_argv[1]=_tmp_argv[1]; tmp_argv[2]=_tmp_argv[2]; tmp_argv[3]=_tmp_argv[3];
strcpybuff(tmp_argv[0],"--");
strcatbuff(tmp_argv[0],a);
strcpybuff(tmp_argv[1],b);
result=optalias_check(2,(const char * const *)tmp_argv,0,
&return_argc,(tmp_argv+2),
return_error);
char _tmp_argv[4][HTS_CDLMAXSIZE];
char *tmp_argv[4];
tmp_argv[0] = _tmp_argv[0];
tmp_argv[1] = _tmp_argv[1];
tmp_argv[2] = _tmp_argv[2];
tmp_argv[3] = _tmp_argv[3];
strcpybuff(tmp_argv[0], "--");
strcatbuff(tmp_argv[0], a);
strcpybuff(tmp_argv[1], b);
result =
optalias_check(2, (const char *const *) tmp_argv, 0, &return_argc,
(tmp_argv + 2), return_error);
if (!result) {
printf("%s\n",return_error);
printf("%s\n", return_error);
} else {
int insert_after_argc;
/* Insert parameters BUT so that they can be in the same order */
/* temporary argc: Number of parameters after minus insert_after_argc */
insert_after_argc=(*argc)-insert_after;
cmdl_ins((tmp_argv[2]),insert_after_argc,(argv+insert_after),x_argvblk,(*x_ptr));
*argc=insert_after_argc+insert_after;
insert_after_argc = (*argc) - insert_after;
cmdl_ins((tmp_argv[2]), insert_after_argc, (argv + insert_after),
x_argvblk, (*x_ptr));
*argc = insert_after_argc + insert_after;
insert_after++;
/* Second one */
if (return_argc>1) {
insert_after_argc=(*argc)-insert_after;
cmdl_ins((tmp_argv[3]),insert_after_argc,(argv+insert_after),x_argvblk,(*x_ptr));
*argc=insert_after_argc+insert_after;
if (return_argc > 1) {
insert_after_argc = (*argc) - insert_after;
cmdl_ins((tmp_argv[3]), insert_after_argc,
(argv + insert_after), x_argvblk, (*x_ptr));
*argc = insert_after_argc + insert_after;
insert_after++;
}
/* increment to nbr of used parameters */
@@ -529,7 +566,7 @@ int optinclude_file(const char* name,
}
}
}
}
}
fclose(fp);
@@ -540,9 +577,10 @@ int optinclude_file(const char* name,
/* Get home directory, '.' if failed */
/* example: /home/smith */
char* hts_gethome(void) {
char *hts_gethome(void) {
#ifndef _WIN32_WCE
char* home = getenv( "HOME" );
char *home = getenv("HOME");
if (home)
return home;
else
@@ -551,9 +589,10 @@ char* hts_gethome(void) {
}
/* Convert ~/foo into /home/smith/foo */
void expand_home(String *str) {
void expand_home(String * str) {
if (StringSub(*str, 1) == '~') {
char BIGSTK tempo[HTS_URLMAXSIZE*2];
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
strcpybuff(tempo, hts_gethome());
strcatbuff(tempo, StringBuff(*str) + 1);
StringCopy(*str, tempo);

33
src/htsalias.h
View File

@@ -17,45 +17,40 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsalias.h subroutines: */
/* alias for command-line options and config files */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSALIAS_DEFH
#define HTSALIAS_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
extern const char* hts_optalias[][4];
int optalias_check(int argc,const char * const * argv,int n_arg,
int* return_argc,char** return_argv,
char* return_error);
int optalias_find(const char* token);
const char* optalias_help(const char* token);
int optreal_find(const char* token);
int optinclude_file(const char* name,
int* argc,char** argv,char* x_argvblk,int* x_ptr);
const char* optreal_value(int p);
const char* optalias_value(int p);
const char* opttype_value(int p);
const char* opthelp_value(int p);
char* hts_gethome(void);
void expand_home(String *str);
extern const char *hts_optalias[][4];
int optalias_check(int argc, const char *const *argv, int n_arg,
int *return_argc, char **return_argv, char *return_error);
int optalias_find(const char *token);
const char *optalias_help(const char *token);
int optreal_find(const char *token);
int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
int *x_ptr);
const char *optreal_value(int p);
const char *optalias_value(int p);
const char *opttype_value(int p);
const char *opthelp_value(int p);
char *hts_gethome(void);
void expand_home(String * str);
#endif
#endif

5032
src/htsback.c
View File

File diff suppressed because it is too large Load Diff

114
src/htsback.h
View File

@@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
@@ -34,9 +32,8 @@ Please visit our Website: http://www.httrack.com
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSBACK_DEFH
#define HTSBACK_DEFH
#define HTSBACK_DEFH
#include "htsglobal.h"
@@ -72,62 +69,79 @@ typedef struct htsblk htsblk;
#ifdef HTS_INTERNAL_BYTECODE
// create/destroy
struct_back* back_new(int back_max);
void back_free(struct_back** sback);
struct_back *back_new(int back_max);
void back_free(struct_back ** sback);
// backing
#define BACK_ADD_TEST "(dummy)"
#define BACK_ADD_TEST2 "(dummy2)"
int back_index(httrackp* opt, struct_back* sback,char* adr,char* fil,char* sav);
int back_available(struct_back* sback);
LLint back_incache(struct_back* sback);
int back_done_incache(struct_back* sback);
HTS_INLINE int back_exist(struct_back* sback,httrackp* opt,char* adr,char* fil,char* sav);
int back_nsoc(struct_back* sback);
int back_nsoc_overall(struct_back* sback);
int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test);
int back_add_if_not_exists(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test);
int back_stack_available(struct_back* sback);
int back_search(httrackp* opt, struct_back* sback);
int back_search_quick(struct_back* sback);
void back_clean(httrackp* opt,cache_back* cache,struct_back* sback);
int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback);
void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_timestart);
int back_letlive(httrackp* opt, cache_back* cache, struct_back* sback, const int p);
int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr);
void back_connxfr(htsblk* src, htsblk* dst);
void back_move(lien_back* src, lien_back* dst);
void back_copy_static(const lien_back* src, lien_back* dst);
int back_serialize(FILE *fp, const lien_back* src);
int back_unserialize(FILE *fp, lien_back** dst);
int back_serialize_ref(httrackp* opt, const lien_back* src);
int back_unserialize_ref(httrackp* opt, const char *adr, const char *fil, lien_back** dst);
void back_set_finished(struct_back* sback, const int p);
void back_set_locked(struct_back* sback, const int p);
void back_set_unlocked(struct_back* sback, const int p);
int back_delete(httrackp* opt,cache_back* cache,struct_back* sback,const int p);
void back_index_unlock(struct_back* sback, const int p);
int back_clear_entry(lien_back* back);
int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, const int p);
void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback);
int back_maydelete(httrackp* opt, cache_back* cache, struct_back* sback, const int p);
void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, const int p);
int back_trylive(httrackp* opt,cache_back* cache,struct_back* sback, const int p);
int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,const int p);
void back_info(struct_back* sback,int i,int j,FILE* fp);
void back_infostr(struct_back* sback,int i,int j,char* s);
LLint back_transfered(LLint add,struct_back* sback);
int back_index(httrackp * opt, struct_back * sback, char *adr, char *fil,
char *sav);
int back_available(struct_back * sback);
LLint back_incache(struct_back * sback);
int back_done_incache(struct_back * sback);
HTS_INLINE int back_exist(struct_back * sback, httrackp * opt, char *adr,
char *fil, char *sav);
int back_nsoc(struct_back * sback);
int back_nsoc_overall(struct_back * sback);
int back_add(struct_back * sback, httrackp * opt, cache_back * cache, char *adr,
char *fil, char *save, char *referer_adr, char *referer_fil,
int test);
int back_add_if_not_exists(struct_back * sback, httrackp * opt,
cache_back * cache, char *adr, char *fil, char *save,
char *referer_adr, char *referer_fil, int test);
int back_stack_available(struct_back * sback);
int back_search(httrackp * opt, struct_back * sback);
int back_search_quick(struct_back * sback);
void back_clean(httrackp * opt, cache_back * cache, struct_back * sback);
int back_cleanup_background(httrackp * opt, cache_back * cache,
struct_back * sback);
void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
TStamp stat_timestart);
int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
const int p);
int back_searchlive(httrackp * opt, struct_back * sback, char *search_addr);
void back_connxfr(htsblk * src, htsblk * dst);
void back_move(lien_back * src, lien_back * dst);
void back_copy_static(const lien_back * src, lien_back * dst);
int back_serialize(FILE * fp, const lien_back * src);
int back_unserialize(FILE * fp, lien_back ** dst);
int back_serialize_ref(httrackp * opt, const lien_back * src);
int back_unserialize_ref(httrackp * opt, const char *adr, const char *fil,
lien_back ** dst);
void back_set_finished(struct_back * sback, const int p);
void back_set_locked(struct_back * sback, const int p);
void back_set_unlocked(struct_back * sback, const int p);
int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
const int p);
void back_index_unlock(struct_back * sback, const int p);
int back_clear_entry(lien_back * back);
int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
const int p);
void back_delete_all(httrackp * opt, cache_back * cache, struct_back * sback);
int back_maydelete(httrackp * opt, cache_back * cache, struct_back * sback,
const int p);
void back_maydeletehttp(httrackp * opt, cache_back * cache, struct_back * sback,
const int p);
int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback,
const int p);
int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
const int p);
void back_info(struct_back * sback, int i, int j, FILE * fp);
void back_infostr(struct_back * sback, int i, int j, char *s);
LLint back_transfered(LLint add, struct_back * sback);
// hostback
#if HTS_XGETHOST
void back_solve(httrackp *opt,lien_back* sback);
int host_wait(httrackp *opt, lien_back* sback);
void back_solve(httrackp * opt, lien_back * sback);
int host_wait(httrackp * opt, lien_back * sback);
#endif
int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize);
int back_checkmirror(httrackp* opt);
int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
int back_checkmirror(httrackp * opt);
#if HTS_XGETHOST
#if USE_BEGINTHREAD
void Hostlookup(void* iadr_p);
void Hostlookup(void *iadr_p);
#endif
#endif

31
src/htsbase.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Basic definitions */
/* Used in .c files for basic (malloc() ..) definitions */
@@ -98,11 +95,9 @@ extern "C" {
// caractère maj
#define isUpperLetter(a) ( ((a) >= 'A') && ((a) <= 'Z') )
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
// functions
#ifdef _WIN32
#define DynamicGet(handle, sym) GetProcAddress(handle, sym)
@@ -111,8 +106,8 @@ extern "C" {
#endif
// emergency log
typedef void (*t_abortLog)(char* msg, char* file, int line);
extern HTSEXT_API t_abortLog abortLog__;
typedef void (*t_abortLog) (char *msg, char *file, int line);
extern HTSEXT_API t_abortLog abortLog__;
#define abortLog(a) abortLog__(a, __FILE__, __LINE__)
#define _ ,
#ifndef _WIN32_WCE
@@ -173,14 +168,14 @@ extern HTSEXT_API t_abortLog abortLog__;
#define strdupt(A) hts_strdup(A)
#define freet(A) do { hts_free(A); (A) = NULL; } while(0)
#define realloct(A,B) hts_realloc(A,B)
void hts_freeall();
void* hts_malloc (size_t);
void* hts_calloc(size_t,size_t);
char* hts_strdup(char*);
void* hts_xmalloc(size_t,size_t);
void hts_free (void*);
void* hts_realloc (void*,size_t);
mlink* hts_find(char* adr);
void hts_freeall();
void *hts_malloc(size_t);
void *hts_calloc(size_t, size_t);
char *hts_strdup(char *);
void *hts_xmalloc(size_t, size_t);
void hts_free(void *);
void *hts_realloc(void *, size_t);
mlink *hts_find(char *adr);
/* protected memcpy */
#define memcpybuff(A, B, N) do { \
mlink* lnk = hts_find((void*)(A)); \
@@ -195,9 +190,9 @@ mlink* hts_find(char* adr);
#endif
typedef void (* htsErrorCallback)(char* msg, char* file, int line);
extern HTSEXT_API htsErrorCallback htsCallbackErr;
extern HTSEXT_API int htsMemoryFastXfr;
typedef void (*htsErrorCallback) (char *msg, char *file, int line);
extern HTSEXT_API htsErrorCallback htsCallbackErr;
extern HTSEXT_API int htsMemoryFastXfr;
/*
*/

53
src/htsbasenet.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Basic net definitions */
/* Used in .c and .h files that needs hostent and so */
@@ -41,7 +38,7 @@ Please visit our Website: http://www.httrack.com
#ifdef _WIN32
#if HTS_INET6==0
#include <winsock2.h>
#include <winsock2.h>
#else
#ifndef _WIN32_WCE
@@ -57,8 +54,8 @@ Please visit our Website: http://www.httrack.com
//#include <winsock2.h>
//#include <tpipv6.h>
#else
#include <winsock2.h>
#include <socket.h>
#include <winsock2.h>
#include <socket.h>
#endif
#endif
@@ -67,8 +64,8 @@ typedef struct hostent FAR t_hostent;
#else
#define HTS_USESCOPEID
#define INVALID_SOCKET -1
typedef struct hostent t_hostent;
#define INVALID_SOCKET -1
typedef struct hostent t_hostent;
#endif
#if HTS_USEOPENSSL
@@ -113,26 +110,26 @@ typedef struct hostent FAR t_hostent;
/* */
typedef void SSL_CTX;
typedef void* SSL;
typedef void *SSL;
typedef void SSL_METHOD;
typedef int (*t_SSL_shutdown)(SSL *ssl);
typedef void (*t_SSL_free)(SSL *ssl);
typedef SSL (*t_SSL_new)(SSL_CTX *ctx);
typedef int (*t_SSL_clear)(SSL *ssl);
typedef int (*t_SSL_set_fd)(SSL *ssl, int fd);
typedef void (*t_SSL_set_connect_state)(SSL *ssl);
typedef int (*t_SSL_connect)(SSL *ssl);
typedef int (*t_SSL_get_error)(SSL *ssl, int ret);
typedef int (*t_SSL_write)(SSL *ssl, const void *buf, int num);
typedef int (*t_SSL_read)(SSL *ssl, void *buf, int num);
typedef int (*t_SSL_library_init)(void);
typedef void (*t_ERR_load_crypto_strings)(void);
typedef void (*t_ERR_load_SSL_strings)(void);
typedef SSL_METHOD * (*t_SSLv23_client_method)(void);
typedef SSL_CTX * (*t_SSL_CTX_new)(SSL_METHOD *method);
typedef char * (*t_ERR_error_string)(unsigned long e, char *buf);
typedef void (*t_SSL_load_error_strings)(void);
typedef long (*t_SSL_CTX_ctrl)(SSL_CTX *ctx, int cmd, long larg, char *parg);
typedef int (*t_SSL_shutdown) (SSL * ssl);
typedef void (*t_SSL_free) (SSL * ssl);
typedef SSL(*t_SSL_new) (SSL_CTX * ctx);
typedef int (*t_SSL_clear) (SSL * ssl);
typedef int (*t_SSL_set_fd) (SSL * ssl, int fd);
typedef void (*t_SSL_set_connect_state) (SSL * ssl);
typedef int (*t_SSL_connect) (SSL * ssl);
typedef int (*t_SSL_get_error) (SSL * ssl, int ret);
typedef int (*t_SSL_write) (SSL * ssl, const void *buf, int num);
typedef int (*t_SSL_read) (SSL * ssl, void *buf, int num);
typedef int (*t_SSL_library_init) (void);
typedef void (*t_ERR_load_crypto_strings) (void);
typedef void (*t_ERR_load_SSL_strings) (void);
typedef SSL_METHOD *(*t_SSLv23_client_method) (void);
typedef SSL_CTX *(*t_SSL_CTX_new) (SSL_METHOD * method);
typedef char *(*t_ERR_error_string) (unsigned long e, char *buf);
typedef void (*t_SSL_load_error_strings) (void);
typedef long (*t_SSL_CTX_ctrl) (SSL_CTX * ctx, int cmd, long larg, char *parg);
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
@@ -167,7 +164,7 @@ From /usr/include/openssl/ssl.h
#define SSL_ERROR_WANT_READ 2
#define SSL_ERROR_WANT_WRITE 3
#define SSL_ERROR_WANT_X509_LOOKUP 4
#define SSL_ERROR_SYSCALL 5 /* look at error stack/return value/errno */
#define SSL_ERROR_SYSCALL 5 /* look at error stack/return value/errno */
#define SSL_ERROR_ZERO_RETURN 6
#define SSL_ERROR_WANT_CONNECT 7
#define SSL_OP_ALL 0x000FFFFFL

413
src/htsbauth.c
View File

@@ -16,18 +16,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* basic authentication: password storage */
@@ -48,73 +45,82 @@ Please visit our Website: http://www.httrack.com
// gestion des cookie
// ajoute, dans l'ordre
// !=0 : erreur
int cookie_add(t_cookie* cookie,char* cook_name,char* cook_value,char* domain,char* path) {
char buffer[8192];
char* a=cookie->data;
char* insert;
int cookie_add(t_cookie * cookie, char *cook_name, char *cook_value,
char *domain, char *path) {
char buffer[8192];
char *a = cookie->data;
char *insert;
char cook[16384];
// effacer éventuel cookie en double
cookie_del(cookie,cook_name,domain,path);
if ((int)strlen(cook_value)>1024) return -1; // trop long
if ((int)strlen(cook_name)>256) return -1; // trop long
if ((int)strlen(domain)>256) return -1; // trop long
if ((int)strlen(path)>256) return -1; // trop long
if ((int)(
strlen(cookie->data)
+strlen(cook_value)
+strlen(cook_name)
+strlen(domain)
+strlen(path)
+256
) > cookie->max_len) return -1; // impossible d'ajouter
insert=a; // insérer ici
while (*a) {
if ( strlen(cookie_get(buffer, a,2)) < strlen(path) ) // long. path (le + long est prioritaire)
a=cookie->data+strlen(cookie->data); // fin
// effacer éventuel cookie en double
cookie_del(cookie, cook_name, domain, path);
if ((int) strlen(cook_value) > 1024)
return -1; // trop long
if ((int) strlen(cook_name) > 256)
return -1; // trop long
if ((int) strlen(domain) > 256)
return -1; // trop long
if ((int) strlen(path) > 256)
return -1; // trop long
if ((int) (strlen(cookie->data)
+ strlen(cook_value)
+ strlen(cook_name)
+ strlen(domain)
+ strlen(path)
+ 256) > cookie->max_len)
return -1; // impossible d'ajouter
insert = a; // insérer ici
while(*a) {
if (strlen(cookie_get(buffer, a, 2)) < strlen(path)) // long. path (le + long est prioritaire)
a = cookie->data + strlen(cookie->data); // fin
else {
a=strchr(a,'\n'); // prochain champ
if (a==NULL)
a=cookie->data+strlen(cookie->data); // fin
a = strchr(a, '\n'); // prochain champ
if (a == NULL)
a = cookie->data + strlen(cookie->data); // fin
else
a++;
while(*a=='\n') a++;
insert=a; // insérer ici
while(*a == '\n')
a++;
insert = a; // insérer ici
}
}
// construction du cookie
strcpybuff(cook,domain);
strcatbuff(cook,"\t");
strcatbuff(cook,"TRUE");
strcatbuff(cook,"\t");
strcatbuff(cook,path);
strcatbuff(cook,"\t");
strcatbuff(cook,"FALSE");
strcatbuff(cook,"\t");
strcatbuff(cook,"1999999999");
strcatbuff(cook,"\t");
strcatbuff(cook,cook_name);
strcatbuff(cook,"\t");
strcatbuff(cook,cook_value);
strcatbuff(cook,"\n");
if (!( ((int) strlen(cookie->data) + (int) strlen(cook)) < cookie->max_len)) return -1; // impossible d'ajouter
cookie_insert(insert,cook);
strcpybuff(cook, domain);
strcatbuff(cook, "\t");
strcatbuff(cook, "TRUE");
strcatbuff(cook, "\t");
strcatbuff(cook, path);
strcatbuff(cook, "\t");
strcatbuff(cook, "FALSE");
strcatbuff(cook, "\t");
strcatbuff(cook, "1999999999");
strcatbuff(cook, "\t");
strcatbuff(cook, cook_name);
strcatbuff(cook, "\t");
strcatbuff(cook, cook_value);
strcatbuff(cook, "\n");
if (!(((int) strlen(cookie->data) + (int) strlen(cook)) < cookie->max_len))
return -1; // impossible d'ajouter
cookie_insert(insert, cook);
#if DEBUG_COOK
printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",cook_name,cook_value,domain,path);
printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
cook_name, cook_value, domain, path);
//printf(">>>cook: %s<<<\n",cookie->data);
#endif
return 0;
}
// effacer cookie si existe
int cookie_del(t_cookie* cookie,char* cook_name,char* domain,char* path) {
char *a,*b;
b=cookie_find(cookie->data,cook_name,domain,path);
int cookie_del(t_cookie * cookie, char *cook_name, char *domain, char *path) {
char *a, *b;
b = cookie_find(cookie->data, cook_name, domain, path);
if (b) {
a=cookie_nextfield(b);
cookie_delete(b,(int) (a - b));
a = cookie_nextfield(b);
cookie_delete(b, (int) (a - b));
#if DEBUG_COOK
printf("deleted old cookie: %s %s %s\n",cook_name,domain,path);
printf("deleted old cookie: %s %s %s\n", cook_name, domain, path);
#endif
}
return 0;
@@ -123,100 +129,112 @@ int cookie_del(t_cookie* cookie,char* cook_name,char* domain,char* path) {
// rechercher cookie à partir de la position s (par exemple s=cookie.data)
// renvoie pointeur sur ligne, ou NULL si introuvable
// path est aligné à droite et cook_name peut être vide (chercher alors tout cookie)
// .doubleclick.net TRUE / FALSE 1999999999 id A
char* cookie_find(char* s,char* cook_name,char* domain,char* path) {
char buffer[8192];
char* a=s;
while (*a) {
// .doubleclick.net TRUE / FALSE 1999999999 id A
char *cookie_find(char *s, char *cook_name, char *domain, char *path) {
char buffer[8192];
char *a = s;
while(*a) {
int t;
if (strnotempty(cook_name)==0)
t=1; // accepter par défaut
if (strnotempty(cook_name) == 0)
t = 1; // accepter par défaut
else
t=( strcmp(cookie_get(buffer, a,5),cook_name)==0 ); // tester si même nom
if (t) { // même nom ou nom qualconque
t = (strcmp(cookie_get(buffer, a, 5), cook_name) == 0); // tester si même nom
if (t) { // même nom ou nom qualconque
//
char* chk_dom=cookie_get(buffer,a,0); // domaine concerné par le cookie
char *chk_dom = cookie_get(buffer, a, 0); // domaine concerné par le cookie
if ((int) strlen(chk_dom) <= (int) strlen(domain)) {
if ( strcmp(chk_dom,domain+strlen(domain)-strlen(chk_dom))==0 ) { // même domaine
if (strcmp(chk_dom, domain + strlen(domain) - strlen(chk_dom)) == 0) { // même domaine
//
char* chk_path=cookie_get(buffer,a,2); // chemin concerné par le cookie
char *chk_path = cookie_get(buffer, a, 2); // chemin concerné par le cookie
if ((int) strlen(chk_path) <= (int) strlen(path)) {
if (strncmp(path,chk_path,strlen(chk_path))==0 ) { // même chemin
if (strncmp(path, chk_path, strlen(chk_path)) == 0) { // même chemin
return a;
}
}
}
}
}
a=cookie_nextfield(a);
a = cookie_nextfield(a);
}
return NULL;
}
// renvoie prochain champ
char* cookie_nextfield(char* a) {
char* b=a;
a=strchr(a,'\n'); // prochain champ
if (a==NULL)
a=b+strlen(b); // fin
char *cookie_nextfield(char *a) {
char *b = a;
a = strchr(a, '\n'); // prochain champ
if (a == NULL)
a = b + strlen(b); // fin
else
a++;
while(*a=='\n') a++;
while(*a == '\n')
a++;
return a;
}
// lire cookies.txt
// lire également (Windows seulement) les *@*.txt (cookies IE copiés)
// !=0 : erreur
int cookie_load(t_cookie* cookie, const char* fpath, const char* name) {
char catbuff[CATBUFF_SIZE];
char buffer[8192];
// cookie->data[0]='\0';
int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
char catbuff[CATBUFF_SIZE];
char buffer[8192];
// cookie->data[0]='\0';
// Fusionner d'abord les éventuels cookies IE
#ifdef _WIN32
{
WIN32_FIND_DATAA find;
HANDLE h;
char pth[MAX_PATH + 32];
strcpybuff(pth,fpath);
strcatbuff(pth,"*@*.txt");
h = FindFirstFileA((char*)pth,&find);
char pth[MAX_PATH + 32];
strcpybuff(pth, fpath);
strcatbuff(pth, "*@*.txt");
h = FindFirstFileA((char *) pth, &find);
if (h != INVALID_HANDLE_VALUE) {
do {
if (!(find.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ))
if (!(find.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM )) {
FILE* fp=fopen(fconcat(catbuff, fpath, find.cFileName),"rb");
if (!(find.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
if (!(find.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM)) {
FILE *fp = fopen(fconcat(catbuff, fpath, find.cFileName), "rb");
if (fp) {
char cook_name[256];
char cook_value[1000];
char domainpathpath[512];
char dummy[512];
//
char domain[256]; // domaine cookie (.netscape.com)
char path[256]; // chemin (/)
int cookie_merged=0;
char domain[256]; // domaine cookie (.netscape.com)
char path[256]; // chemin (/)
int cookie_merged = 0;
//
// Read all cookies
while( ! feof(fp) ) {
cook_name[0] = cook_value[0] = domainpathpath[0]
while(!feof(fp)) {
cook_name[0] = cook_value[0] = domainpathpath[0]
= dummy[0] = domain[0] = path[0] = '\0';
linput(fp,cook_name,250);
if ( ! feof(fp) ) {
linput(fp,cook_value,250);
if ( ! feof(fp) ) {
linput(fp, cook_name, 250);
if (!feof(fp)) {
linput(fp, cook_value, 250);
if (!feof(fp)) {
int i;
linput(fp,domainpathpath,500);
linput(fp, domainpathpath, 500);
/* Read 6 other useless values */
for(i = 0 ; ! feof(fp) && i < 6 ; i++) {
linput(fp,dummy,500);
for(i = 0; !feof(fp) && i < 6; i++) {
linput(fp, dummy, 500);
}
if (strnotempty(cook_name)
&& strnotempty(cook_value)
&& strnotempty(domainpathpath)) {
if (ident_url_absolute(domainpathpath,domain,path)>=0) {
cookie_add(cookie,cook_name,cook_value,domain,path);
cookie_merged=1;
if (strnotempty(cook_name)
&& strnotempty(cook_value)
&& strnotempty(domainpathpath)) {
if (ident_url_absolute(domainpathpath, domain, path) >= 0) {
cookie_add(cookie, cook_name, cook_value, domain, path);
cookie_merged = 1;
}
}
}
@@ -224,37 +242,40 @@ int cookie_load(t_cookie* cookie, const char* fpath, const char* name) {
}
fclose(fp);
if (cookie_merged)
remove(fconcat(catbuff,fpath,find.cFileName));
} // if fp
remove(fconcat(catbuff, fpath, find.cFileName));
} // if fp
}
} while(FindNextFileA(h,&find));
} while(FindNextFileA(h, &find));
FindClose(h);
}
}
#endif
// Ensuite, cookies.txt
{
FILE* fp = fopen(fconcat(catbuff, fpath, name),"rb");
FILE *fp = fopen(fconcat(catbuff, fpath, name), "rb");
if (fp) {
char BIGSTK line[8192];
while( (!feof(fp)) && (((int) strlen(cookie->data)) < cookie->max_len)) {
rawlinput(fp,line,8100);
while((!feof(fp)) && (((int) strlen(cookie->data)) < cookie->max_len)) {
rawlinput(fp, line, 8100);
if (strnotempty(line)) {
if (strlen(line)<8000) {
if (line[0]!='#') {
char domain[256]; // domaine cookie (.netscape.com)
char path[256]; // chemin (/)
char cook_name[1024]; // nom cookie (MYCOOK)
char BIGSTK cook_value[8192]; // valeur (ID=toto,S=1234)
strcpybuff(domain,cookie_get(buffer,line,0)); // host
strcpybuff(path,cookie_get(buffer,line,2)); // path
strcpybuff(cook_name,cookie_get(buffer,line,5)); // name
strcpybuff(cook_value,cookie_get(buffer,line,6)); // value
if (strlen(line) < 8000) {
if (line[0] != '#') {
char domain[256]; // domaine cookie (.netscape.com)
char path[256]; // chemin (/)
char cook_name[1024]; // nom cookie (MYCOOK)
char BIGSTK cook_value[8192]; // valeur (ID=toto,S=1234)
strcpybuff(domain, cookie_get(buffer, line, 0)); // host
strcpybuff(path, cookie_get(buffer, line, 2)); // path
strcpybuff(cook_name, cookie_get(buffer, line, 5)); // name
strcpybuff(cook_value, cookie_get(buffer, line, 6)); // value
#if DEBUG_COOK
printf("%s\n",line);
printf("%s\n", line);
#endif
cookie_add(cookie,cook_name,cook_value,domain,path);
cookie_add(cookie, cook_name, cook_value, domain, path);
}
}
}
@@ -268,17 +289,22 @@ int cookie_load(t_cookie* cookie, const char* fpath, const char* name) {
// écrire cookies.txt
// !=0 : erreur
int cookie_save(t_cookie* cookie,char* name) {
char catbuff[CATBUFF_SIZE];
int cookie_save(t_cookie * cookie, char *name) {
char catbuff[CATBUFF_SIZE];
if (strnotempty(cookie->data)) {
char BIGSTK line[8192];
FILE* fp = fopen(fconv(catbuff,name),"wb");
FILE *fp = fopen(fconv(catbuff, name), "wb");
if (fp) {
char* a=cookie->data;
fprintf(fp,"# HTTrack Website Copier Cookie File"LF"# This file format is compatible with Netscape cookies"LF);
char *a = cookie->data;
fprintf(fp,
"# HTTrack Website Copier Cookie File" LF
"# This file format is compatible with Netscape cookies" LF);
do {
a+=binput(a,line,8000);
fprintf(fp,"%s"LF,line);
a += binput(a, line, 8000);
fprintf(fp, "%s" LF, line);
} while(strnotempty(line));
fclose(fp);
return 0;
@@ -289,30 +315,33 @@ int cookie_save(t_cookie* cookie,char* name) {
}
// insertion chaine ins avant s
void cookie_insert(char* s,char* ins) {
char* buff;
if (strnotempty(s)==0) { // rien à faire, juste concat
strcatbuff(s,ins);
void cookie_insert(char *s, char *ins) {
char *buff;
if (strnotempty(s) == 0) { // rien à faire, juste concat
strcatbuff(s, ins);
} else {
buff=(char*) malloct(strlen(s)+2);
buff = (char *) malloct(strlen(s) + 2);
if (buff) {
strcpybuff(buff,s); // copie temporaire
strcpybuff(s,ins); // insérer
strcatbuff(s,buff); // copier
strcpybuff(buff, s); // copie temporaire
strcpybuff(s, ins); // insérer
strcatbuff(s, buff); // copier
freet(buff);
}
}
}
// destruction chaine dans s position pos
void cookie_delete(char* s,int pos) {
char* buff;
if (strnotempty(s+pos)==0) { // rien à faire, effacer
s[0]='\0';
void cookie_delete(char *s, int pos) {
char *buff;
if (strnotempty(s + pos) == 0) { // rien à faire, effacer
s[0] = '\0';
} else {
buff=(char*) malloct(strlen(s+pos)+2);
buff = (char *) malloct(strlen(s + pos) + 2);
if (buff) {
strcpybuff(buff,s+pos); // copie temporaire
strcpybuff(s,buff); // copier
strcpybuff(buff, s + pos); // copie temporaire
strcpybuff(s, buff); // copier
freet(buff);
}
}
@@ -320,28 +349,34 @@ void cookie_delete(char* s,int pos) {
// renvoie champ param de la chaine cookie_base
// ex: cookie_get("ceci est<tab>un<tab>exemple",1) renvoi "un"
char* cookie_get(char *buffer,char* cookie_base,int param) {
char * limit;
char *cookie_get(char *buffer, char *cookie_base, int param) {
char *limit;
while(*cookie_base=='\n') cookie_base++;
limit = strchr(cookie_base,'\n');
if (!limit) limit=cookie_base+strlen(cookie_base);
while(*cookie_base == '\n')
cookie_base++;
limit = strchr(cookie_base, '\n');
if (!limit)
limit = cookie_base + strlen(cookie_base);
if (limit) {
if (param) {
int i;
for(i=0;i<param;i++) {
for(i = 0; i < param; i++) {
if (cookie_base) {
cookie_base=strchr(cookie_base,'\t'); // prochain tab
if (cookie_base) cookie_base++;
cookie_base = strchr(cookie_base, '\t'); // prochain tab
if (cookie_base)
cookie_base++;
}
}
}
if (cookie_base) {
if ( cookie_base < limit) {
char* a = cookie_base;
while( (*a) && (*a!='\t') && (*a!='\n')) a++;
buffer[0]='\0';
strncatbuff(buffer,cookie_base,(int) (a - cookie_base));
if (cookie_base < limit) {
char *a = cookie_base;
while((*a) && (*a != '\t') && (*a != '\n'))
a++;
buffer[0] = '\0';
strncatbuff(buffer, cookie_base, (int) (a - cookie_base));
return buffer;
} else
return "";
@@ -350,28 +385,29 @@ char* cookie_get(char *buffer,char* cookie_base,int param) {
} else
return "";
}
// fin cookies
// -- basic auth --
/* déclarer un répertoire comme possédant une authentification propre */
int bauth_add(t_cookie* cookie,char* adr,char* fil,char* auth) {
char buffer[HTS_URLMAXSIZE*2];
if (cookie) {
if (!bauth_check(cookie,adr,fil)) { // n'existe pas déja
bauth_chain* chain=&cookie->auth;
char* prefix=bauth_prefix(buffer,adr,fil);
int bauth_add(t_cookie * cookie, char *adr, char *fil, char *auth) {
char buffer[HTS_URLMAXSIZE * 2];
if (cookie) {
if (!bauth_check(cookie, adr, fil)) { // n'existe pas déja
bauth_chain *chain = &cookie->auth;
char *prefix = bauth_prefix(buffer, adr, fil);
/* fin de la chaine */
while(chain->next)
chain=chain->next;
chain->next=(bauth_chain*) calloc(sizeof(bauth_chain),1);
chain = chain->next;
chain->next = (bauth_chain *) calloc(sizeof(bauth_chain), 1);
if (chain->next) {
chain=chain->next;
chain->next=NULL;
strcpybuff(chain->auth,auth);
strcpybuff(chain->prefix,prefix);
chain = chain->next;
chain->next = NULL;
strcpybuff(chain->auth, auth);
strcpybuff(chain->prefix, prefix);
return 1;
}
}
@@ -381,33 +417,38 @@ int bauth_add(t_cookie* cookie,char* adr,char* fil,char* auth) {
/* tester adr et fil, et retourner authentification si nécessaire */
/* sinon, retourne NULL */
char* bauth_check(t_cookie* cookie,char* adr,char* fil) {
char buffer[HTS_URLMAXSIZE*2];
char *bauth_check(t_cookie * cookie, char *adr, char *fil) {
char buffer[HTS_URLMAXSIZE * 2];
if (cookie) {
bauth_chain* chain=&cookie->auth;
char* prefix=bauth_prefix(buffer,adr,fil);
bauth_chain *chain = &cookie->auth;
char *prefix = bauth_prefix(buffer, adr, fil);
while(chain) {
if (strnotempty(chain->prefix)) {
if (strncmp(prefix,chain->prefix,strlen(chain->prefix))==0) {
if (strncmp(prefix, chain->prefix, strlen(chain->prefix)) == 0) {
return chain->auth;
}
}
chain=chain->next;
chain = chain->next;
}
}
return NULL;
}
char* bauth_prefix(char *prefix,char* adr,char* fil) {
char* a;
strcpybuff(prefix,jump_identification(adr));
strcatbuff(prefix,fil);
a=strchr(prefix,'?');
if (a) *a='\0';
if (strchr(prefix,'/')) {
a=prefix+strlen(prefix)-1;
while(*a != '/') a--;
*(a+1)='\0';
char *bauth_prefix(char *prefix, char *adr, char *fil) {
char *a;
strcpybuff(prefix, jump_identification(adr));
strcatbuff(prefix, fil);
a = strchr(prefix, '?');
if (a)
*a = '\0';
if (strchr(prefix, '/')) {
a = prefix + strlen(prefix) - 1;
while(*a != '/')
a--;
*(a + 1) = '\0';
}
return prefix;
}

39
src/htsbauth.h
View File

@@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
@@ -34,10 +32,8 @@ Please visit our Website: http://www.httrack.com
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSBAUTH_DEFH
#define HTSBAUTH_DEFH
#define HTSBAUTH_DEFH
// robots wizard
#ifndef HTS_DEF_FWSTRUCT_bauth_chain
@@ -45,12 +41,11 @@ Please visit our Website: http://www.httrack.com
typedef struct bauth_chain bauth_chain;
#endif
struct bauth_chain {
char prefix[1024]; /* www.foo.com/secure/ */
char auth[1024]; /* base-64 encoded user:pass */
struct bauth_chain* next; /* next element */
char prefix[1024]; /* www.foo.com/secure/ */
char auth[1024]; /* base-64 encoded user:pass */
struct bauth_chain *next; /* next element */
};
// buffer pour les cookies et authentification
#ifndef HTS_DEF_FWSTRUCT_t_cookie
#define HTS_DEF_FWSTRUCT_t_cookie
@@ -62,25 +57,25 @@ struct t_cookie {
bauth_chain auth;
};
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
// cookies
int cookie_add(t_cookie* cookie,char* cook_name,char* cook_value,char* domain,char* path);
int cookie_del(t_cookie* cookie,char* cook_name,char* domain,char* path);
int cookie_load(t_cookie* cookie, const char* path, const char* name);
int cookie_save(t_cookie* cookie,char* name);
void cookie_insert(char* s,char* ins);
void cookie_delete(char* s,int pos);
char* cookie_get(char *buffer, char* cookie_base,int param);
char* cookie_find(char* s,char* cook_name,char* domain,char* path);
char* cookie_nextfield(char* a);
int cookie_add(t_cookie * cookie, char *cook_name, char *cook_value,
char *domain, char *path);
int cookie_del(t_cookie * cookie, char *cook_name, char *domain, char *path);
int cookie_load(t_cookie * cookie, const char *path, const char *name);
int cookie_save(t_cookie * cookie, char *name);
void cookie_insert(char *s, char *ins);
void cookie_delete(char *s, int pos);
char *cookie_get(char *buffer, char *cookie_base, int param);
char *cookie_find(char *s, char *cook_name, char *domain, char *path);
char *cookie_nextfield(char *a);
// basic auth
int bauth_add(t_cookie* cookie,char* adr,char* fil,char* auth);
char* bauth_check(t_cookie* cookie,char* adr,char* fil);
char* bauth_prefix(char *buffer, char* adr,char* fil);
int bauth_add(t_cookie * cookie, char *adr, char *fil, char *auth);
char *bauth_check(t_cookie * cookie, char *adr, char *fil);
char *bauth_prefix(char *buffer, char *adr, char *fil);
#endif

2058
src/htscache.c
View File

File diff suppressed because it is too large Load Diff

59
src/htscache.h
View File

@@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* cache system (index and stores files in cache) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSCACHE_DEFH
#define HTSCACHE_DEFH
#define HTSCACHE_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
@@ -62,29 +57,41 @@ typedef struct htsblk htsblk;
#endif
// cache
void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,const char* url_adr,const char* url_fil,const char* url_save);
void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_adr,const char* url_fil,const char* url_save,int all_in_cache,const char* path_prefix);
htsblk cache_read(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location);
htsblk cache_read_ro(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location);
htsblk cache_read_including_broken(httrackp* opt,cache_back* cache,const char* adr,const char* fil);
htsblk cache_readex(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location,char* return_save,int readonly);
htsblk* cache_header(httrackp* opt,cache_back* cache,const char* adr,const char* fil,htsblk* r);
void cache_init(cache_back* cache,httrackp* opt);
void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r,
const char *url_adr, const char *url_fil,
const char *url_save);
void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
const char *url_adr, const char *url_fil, const char *url_save,
int all_in_cache, const char *path_prefix);
htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
const char *fil, const char *save, char *location);
htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
const char *fil, const char *save, char *location);
htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
const char *adr, const char *fil);
htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
const char *fil, const char *save, char *location,
char *return_save, int readonly);
htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
const char *fil, htsblk * r);
void cache_init(cache_back * cache, httrackp * opt);
int cache_writedata(FILE* cache_ndx,FILE* cache_dat,const char* str1,const char* str2,char* outbuff,int len);
int cache_readdata(cache_back* cache,const char* str1,const char* str2,char** inbuff,int* len);
int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
const char *str2, char *outbuff, int len);
int cache_readdata(cache_back * cache, const char *str1, const char *str2,
char **inbuff, int *len);
void cache_rstr(FILE* fp,char* s);
char* cache_rstr_addr(FILE* fp);
int cache_brstr(char* adr,char* s);
int cache_quickbrstr(char* adr,char* s);
int cache_brint(char* adr,int* i);
void cache_rint(FILE* fp,int* i);
void cache_rLLint(FILE* fp,LLint* i);
void cache_rstr(FILE * fp, char *s);
char *cache_rstr_addr(FILE * fp);
int cache_brstr(char *adr, char *s);
int cache_quickbrstr(char *adr, char *s);
int cache_brint(char *adr, int *i);
void cache_rint(FILE * fp, int *i);
void cache_rLLint(FILE * fp, LLint * i);
int cache_wstr(FILE* fp,const char* s);
int cache_wint(FILE* fp,int i);
int cache_wLLint(FILE* fp,LLint i);
int cache_wstr(FILE * fp, const char *s);
int cache_wint(FILE * fp, int i);
int cache_wLLint(FILE * fp, LLint i);
#endif

193
src/htscatchurl.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: URL catch .h */
/* Author: Xavier Roche */
@@ -62,57 +59,60 @@ Please visit our Website: http://www.httrack.com
/* catch url */
#include "htscatchurl.h"
// URL Link catcher
// 0- Init the URL catcher with standard port
// catch_url_init(&port,&return_host);
HTSEXT_API T_SOC catch_url_init_std(int* port_prox,char* adr_prox) {
HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox) {
T_SOC soc;
int try_to_listen_to[]={8080,3128,80,81,82,8081,3129,31337,0,-1};
int i=0;
int try_to_listen_to[] = { 8080, 3128, 80, 81, 82, 8081, 3129, 31337, 0, -1 };
int i = 0;
do {
soc=catch_url_init(&try_to_listen_to[i],adr_prox);
*port_prox=try_to_listen_to[i];
soc = catch_url_init(&try_to_listen_to[i], adr_prox);
*port_prox = try_to_listen_to[i];
i++;
} while( (soc == INVALID_SOCKET) && (try_to_listen_to[i]>=0));
} while((soc == INVALID_SOCKET) && (try_to_listen_to[i] >= 0));
return soc;
}
// 1- Init the URL catcher
// catch_url_init(&port,&return_host);
HTSEXT_API T_SOC catch_url_init(int* port,char* adr) {
HTSEXT_API T_SOC catch_url_init(int *port, char *adr) {
T_SOC soc = INVALID_SOCKET;
char h_loc[256+2];
char h_loc[256 + 2];
if (gethostname(h_loc,256)==0) { // host name
if (gethostname(h_loc, 256) == 0) { // host name
SOCaddr server;
int server_size=sizeof(server);
t_hostent* hp_loc;
int server_size = sizeof(server);
t_hostent *hp_loc;
t_fullhostent buffer;
// effacer structure
memset(&server, 0, sizeof(server));
if ( (hp_loc=vxgethostbyname(h_loc, &buffer)) ) { // notre host
if ((hp_loc = vxgethostbyname(h_loc, &buffer))) { // notre host
// copie adresse
SOCaddr_copyaddr(server, server_size, hp_loc->h_addr_list[0], hp_loc->h_length);
SOCaddr_copyaddr(server, server_size, hp_loc->h_addr_list[0],
hp_loc->h_length);
if ( (soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0)) != INVALID_SOCKET) {
if ((soc =
(T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM,
0)) != INVALID_SOCKET) {
SOCaddr_initport(server, *port);
if ( bind(soc,(struct sockaddr*) &server,server_size) == 0 ) {
if (bind(soc, (struct sockaddr *) &server, server_size) == 0) {
SOCaddr server2;
SOClen len;
len=sizeof(server2);
len = sizeof(server2);
// effacer structure
memset(&server2, 0, sizeof(server2));
if (getsockname(soc,(struct sockaddr*) &server2,&len) == 0) {
*port=ntohs(SOCaddr_sinport(server)); // récupérer port
if (listen(soc,10)>=0) { // au pif le 10
if (getsockname(soc, (struct sockaddr *) &server2, &len) == 0) {
*port = ntohs(SOCaddr_sinport(server)); // récupérer port
if (listen(soc, 10) >= 0) { // au pif le 10
SOCaddr_inetntoa(adr, 128, server2, len);
} else {
#ifdef _WIN32
@@ -120,27 +120,25 @@ HTSEXT_API T_SOC catch_url_init(int* port,char* adr) {
#else
close(soc);
#endif
soc=INVALID_SOCKET;
soc = INVALID_SOCKET;
}
} else {
#ifdef _WIN32
closesocket(soc);
#else
close(soc);
#endif
soc=INVALID_SOCKET;
soc = INVALID_SOCKET;
}
} else {
#ifdef _WIN32
closesocket(soc);
#else
close(soc);
#endif
soc=INVALID_SOCKET;
soc = INVALID_SOCKET;
}
}
}
@@ -154,31 +152,34 @@ HTSEXT_API T_SOC catch_url_init(int* port,char* adr) {
// returns 0 if error
// url: buffer where URL must be stored - or ip:port in case of failure
// data: 32Kb
HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data) {
int retour=0;
HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data) {
int retour = 0;
// connexion (accept)
if (soc != INVALID_SOCKET) {
T_SOC soc2;
while ( (soc2 = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET);
/*
#ifdef _WIN32
closesocket(soc);
#else
close(soc);
#endif
*/
while((soc2 = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET) ;
/*
#ifdef _WIN32
closesocket(soc);
#else
close(soc);
#endif
*/
soc = soc2;
/* INFOS */
{
SOCaddr server2;
SOClen len = sizeof(server2);
// effacer structure
memset(&server2, 0, sizeof(server2));
if (getpeername(soc,(struct sockaddr*) &server2,&len) == 0) {
char dot[256+2];
if (getpeername(soc, (struct sockaddr *) &server2, &len) == 0) {
char dot[256 + 2];
SOCaddr_inetntoa(dot, 256, server2, sizeof(server2));
sprintf(url,"%s:%d", dot, htons(SOCaddr_sinport(server2)));
sprintf(url, "%s:%d", dot, htons(SOCaddr_sinport(server2)));
}
}
/* INFOS */
@@ -187,65 +188,71 @@ HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data) {
if (soc != INVALID_SOCKET) {
char line[1000];
char protocol[256];
line[0]=protocol[0]='\0';
line[0] = protocol[0] = '\0';
//
socinput(soc,line,1000);
socinput(soc, line, 1000);
if (strnotempty(line)) {
if (sscanf(line,"%s %s %s",method,url,protocol) == 3) {
char BIGSTK url_adr[HTS_URLMAXSIZE*2];
char BIGSTK url_fil[HTS_URLMAXSIZE*2];
if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
char BIGSTK url_adr[HTS_URLMAXSIZE * 2];
char BIGSTK url_fil[HTS_URLMAXSIZE * 2];
// méthode en majuscule
int i,r=0;
url_adr[0]=url_fil[0]='\0';
int i, r = 0;
url_adr[0] = url_fil[0] = '\0';
//
for(i=0;i<(int) strlen(method);i++) {
if ((method[i]>='a') && (method[i]<='z'))
method[i]-=('a'-'A');
for(i = 0; i < (int) strlen(method); i++) {
if ((method[i] >= 'a') && (method[i] <= 'z'))
method[i] -= ('a' - 'A');
}
// adresse du lien
if (ident_url_absolute(url,url_adr,url_fil)>=0) {
if (ident_url_absolute(url, url_adr, url_fil) >= 0) {
// Traitement des en-têtes
char BIGSTK loc[HTS_URLMAXSIZE*2];
char BIGSTK loc[HTS_URLMAXSIZE * 2];
htsblk blkretour;
hts_init_htsblk(&blkretour);
//memset(&blkretour, 0, sizeof(htsblk)); // effacer
blkretour.location=loc; // si non nul, contiendra l'adresse véritable en cas de moved xx
blkretour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx
// Lire en têtes restants
sprintf(data,"%s %s %s\r\n",method,url_fil,protocol);
sprintf(data, "%s %s %s\r\n", method, url_fil, protocol);
while(strnotempty(line)) {
socinput(soc,line,1000);
treathead(NULL,NULL,NULL,&blkretour,line); // traiter
strcatbuff(data,line);
strcatbuff(data,"\r\n");
socinput(soc, line, 1000);
treathead(NULL, NULL, NULL, &blkretour, line); // traiter
strcatbuff(data, line);
strcatbuff(data, "\r\n");
}
// CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
//strcatbuff(data,"\r\n");
if (blkretour.totalsize>0) {
int len=(int)min(blkretour.totalsize,32000);
if (blkretour.totalsize > 0) {
int len = (int) min(blkretour.totalsize, 32000);
int pos = (int) strlen(data);
// Copier le reste (post éventuel)
while((len>0) && ((r=recv(soc,(char*) data+pos,len,0))>0) ) {
pos+=r;
len-=r;
data[pos]='\0'; // terminer par NULL
while((len > 0)
&& ((r = recv(soc, (char *) data + pos, len, 0)) > 0)) {
pos += r;
len -= r;
data[pos] = '\0'; // terminer par NULL
}
}
// Envoyer page
sprintf(line,CATCH_RESPONSE);
send(soc,line,(int)strlen(line),0);
sprintf(line, CATCH_RESPONSE);
send(soc, line, (int) strlen(line), 0);
// OK!
retour=1;
retour = 1;
}
}
} // sinon erreur
} // sinon erreur
}
}
if (soc != INVALID_SOCKET) {
#ifdef _WIN32
closesocket(soc);
/*
WSACleanup();
*/
WSACleanup();
*/
#else
close(soc);
#endif
@@ -253,25 +260,31 @@ HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data) {
return retour;
}
// Lecture de ligne sur socket
void socinput(T_SOC soc,char* s,int max) {
void socinput(T_SOC soc, char *s, int max) {
int c;
int j=0;
int j = 0;
do {
unsigned char b;
if (recv(soc,(char*) &b,1,0)==1) {
c=b;
switch(c) {
case 13: break; // sauter CR
case 10: c=-1; break;
case 9: case 12: break; // sauter ces caractères
default: s[j++]=(char) c; break;
if (recv(soc, (char *) &b, 1, 0) == 1) {
c = b;
switch (c) {
case 13:
break; // sauter CR
case 10:
c = -1;
break;
case 9:
case 12:
break; // sauter ces caractères
default:
s[j++] = (char) c;
break;
}
} else
c=EOF;
} while((c!=-1) && (c!=EOF) && (j<(max-1)));
s[j++]='\0';
c = EOF;
} while((c != -1) && (c != EOF) && (j < (max - 1)));
s[j++] = '\0';
}

137
src/htscharset.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Charset conversion functions */
/* Author: Xavier Roche */
@@ -39,8 +36,10 @@ Please visit our Website: http://www.httrack.com
static int hts_isStringAscii(const char *s, size_t size) {
size_t i;
for(i = 0 ; i < size ; i++) {
for(i = 0; i < size; i++) {
const unsigned char c = (const unsigned char) s[i];
if (c >= 0x80) {
return 0;
}
@@ -205,16 +204,19 @@ static const wincodepage_t codepages[] = {
/* Get a Windows codepage, by its name. Return 0 upon error. */
UINT hts_getCodepage(const char *name) {
int id;
#define IS_ALNUM(C) ( ((C) >= 'A' && (C) <= 'Z') || ((C) >= 'a' && (C) <= 'z') || ((C) >= '0' && (C) <= '9') )
#define CHAR_LOWER(C) ( ((C) >= 'A' && (C) <= 'Z') ? ((C) + 'a' - 'A') : (C) )
for(id = 0 ; codepages[id].name != NULL ; id++) {
for(id = 0; codepages[id].name != NULL; id++) {
int i, j;
/* Compare the two strings, lowercase and alphanum only (ISO88591 == iso-8859-1) */
const char *a = name, *b = codepages[id].name;
for(i = 0, j = 0 ; ; i++, j++) {
for(i = 0, j = 0;; i++, j++) {
/* Skip non-alnum */
for( ; a[i] != '\0' && !IS_ALNUM(a[i]) ; i++) ;
for( ; b[j] != '\0' && !IS_ALNUM(b[j]) ; j++) ;
for(; a[i] != '\0' && !IS_ALNUM(a[i]); i++) ;
for(; b[j] != '\0' && !IS_ALNUM(b[j]); j++) ;
/* Compare */
if (CHAR_LOWER(a[i]) != CHAR_LOWER(b[j])) {
break;
@@ -233,6 +235,7 @@ UINT hts_getCodepage(const char *name) {
static char *strndup(const char *s, size_t size) {
char *dest = malloc(size + 1);
if (dest != NULL) {
memcpy(dest, s, size);
dest[size] = '\0';
@@ -241,14 +244,19 @@ static char *strndup(const char *s, size_t size) {
return NULL;
}
LPWSTR hts_convertStringToUCS2(const char *s, int size, UINT cp, int* pwsize) {
LPWSTR hts_convertStringToUCS2(const char *s, int size, UINT cp, int *pwsize) {
/* Size in wide chars of the output */
const int wsize = MultiByteToWideChar(cp, 0, (LPCSTR) s, size, NULL, 0);
if (wsize > 0) {
LPSTR uoutput = NULL;
LPWSTR woutput = malloc((wsize + 1)*sizeof(WCHAR));
if (woutput != NULL && MultiByteToWideChar(cp, 0, (LPCSTR) s, size, woutput, wsize) == wsize) {
const int usize = WideCharToMultiByte(CP_UTF8, 0, woutput, wsize, NULL, 0, NULL, FALSE);
LPWSTR woutput = malloc((wsize + 1) * sizeof(WCHAR));
if (woutput != NULL
&& MultiByteToWideChar(cp, 0, (LPCSTR) s, size, woutput,
wsize) == wsize) {
const int usize =
WideCharToMultiByte(CP_UTF8, 0, woutput, wsize, NULL, 0, NULL, FALSE);
if (usize > 0) {
woutput[wsize] = 0x0;
if (pwsize != NULL)
@@ -262,16 +270,19 @@ LPWSTR hts_convertStringToUCS2(const char *s, int size, UINT cp, int* pwsize) {
return NULL;
}
LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int* pwsize) {
LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int *pwsize) {
return hts_convertStringToUCS2(s, size, CP_UTF8, pwsize);
}
char *hts_convertUCS2StringToCP(LPWSTR woutput, int wsize, UINT cp) {
const int usize = WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, FALSE);
const int usize =
WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, FALSE);
if (usize > 0) {
char *const uoutput = malloc((usize + 1)*sizeof(char));
char *const uoutput = malloc((usize + 1) * sizeof(char));
if (uoutput != NULL) {
if (WideCharToMultiByte(cp, 0, woutput, wsize, uoutput, usize, NULL, FALSE) == usize) {
if (WideCharToMultiByte
(cp, 0, woutput, wsize, uoutput, usize, NULL, FALSE) == usize) {
uoutput[usize] = '\0';
return uoutput;
} else {
@@ -300,8 +311,10 @@ char *hts_convertStringCPToUTF8(const char *s, size_t size, UINT cp) {
/* Size in wide chars of the output */
int wsize;
LPWSTR woutput = hts_convertStringToUCS2(s, (int) size, cp, &wsize);
if (woutput != NULL) {
char *const uoutput = hts_convertUCS2StringToUTF8(woutput, wsize);
free(woutput);
return uoutput;
}
@@ -325,8 +338,10 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
/* Size in wide chars of the output */
int wsize;
LPWSTR woutput = hts_convertStringToUCS2(s, (int) size, CP_UTF8, &wsize);
if (woutput != NULL) {
char *const uoutput = hts_convertUCS2StringToCP(woutput, wsize, cp);
free(woutput);
return uoutput;
}
@@ -338,11 +353,13 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
const UINT cp = hts_getCodepage(charset);
return hts_convertStringCPToUTF8(s, size, cp);
}
char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset) {
const UINT cp = hts_getCodepage(charset);
return hts_convertStringCPFromUTF8(s, size, cp);
}
@@ -355,7 +372,8 @@ char *hts_convertStringSystemToUTF8(const char *s, size_t size) {
#include <errno.h>
#include <iconv.h>
static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to, const char *from) {
static char *hts_convertStringToUTF8_(const char *s, size_t size,
const char *to, const char *from) {
/* Empty string ? */
if (size == 0) {
return strdup("");
@@ -367,8 +385,9 @@ static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to
/* Find codepage */
else {
const iconv_t cp = iconv_open(to, from);
if (cp != (iconv_t) -1) {
char *inbuf = (char*) s;
if (cp != (iconv_t) - 1) {
char *inbuf = (char *) s;
size_t inbytesleft = size;
size_t outbufCapa = 0;
char *outbuf = NULL;
@@ -376,7 +395,7 @@ static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to
size_t finalSize;
/* Initial size to around the string size */
for(outbufCapa = 16 ; outbufCapa < size + 1 ; outbufCapa *= 2) ;
for(outbufCapa = 16; outbufCapa < size + 1; outbufCapa *= 2) ;
outbuf = malloc(outbufCapa);
outbytesleft = outbufCapa;
@@ -384,10 +403,12 @@ static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to
while(outbuf != NULL && inbytesleft != 0) {
const size_t offset = outbufCapa - outbytesleft;
char *outbufCurrent = outbuf + offset;
const size_t ret = iconv(cp, &inbuf, &inbytesleft, &outbufCurrent, &outbytesleft);
if (ret == (size_t) -1) {
const size_t ret =
iconv(cp, &inbuf, &inbytesleft, &outbufCurrent, &outbytesleft);
if (ret == (size_t) - 1) {
if (errno == E2BIG) {
const size_t used = outbufCapa - outbytesleft;
outbufCapa *= 2;
outbuf = realloc(outbuf, outbufCapa);
if (outbuf == NULL) {
@@ -430,7 +451,8 @@ char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
return strdup("");
}
/* Already UTF-8 ? */
if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0 || hts_isStringAscii(s, size)) {
if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0
|| hts_isStringAscii(s, size)) {
return strndup(s, size);
}
/* Find codepage */
@@ -445,7 +467,8 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
return strdup("");
}
/* Already UTF-8 ? */
if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0 || hts_isStringAscii(s, size)) {
if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0
|| hts_isStringAscii(s, size)) {
return strndup(s, size);
}
/* Find codepage */
@@ -456,25 +479,32 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
#endif
HTS_STATIC char* hts_getCharsetFromContentType(const char *mime) {
HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
/* text/html; charset=utf-8 */
const char *const charset = "charset";
char *pos = strstr(mime, charset);
if (pos != NULL) {
/* Skip spaces */
int eq = 0;
for(pos += strlen(charset) ; *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\'' ; pos++) {
for(pos += strlen(charset);
*pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
if (*pos == '=') {
eq = 1;
}
}
if (eq == 1) {
int len;
for(len = 0 ; pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'' ; pos++) ;
for(len = 0;
pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
pos++) ;
if (len != 0) {
char *const s = malloc(len + 1);
int i;
for(i = 0 ; i < len ; i++) {
for(i = 0; i < len; i++) {
s[i] = pos[i];
}
s[len] = '\0';
@@ -503,46 +533,60 @@ static int is_space_or_equal_or_quote(char c) {
}
size_t hts_stringLengthUTF8(const char *s) {
const unsigned char *const bytes = (const unsigned char*) s;
const unsigned char *const bytes = (const unsigned char *) s;
size_t i, len;
for(i = 0, len = 0 ; bytes[i] != '\0' ; i++) {
for(i = 0, len = 0; bytes[i] != '\0'; i++) {
const unsigned char c = bytes[i];
if (HTS_IS_LEADING_UTF8(c)) { // ASCII or leading byte
if (HTS_IS_LEADING_UTF8(c)) { // ASCII or leading byte
len++;
}
}
return len;
}
char* hts_getCharsetFromMeta(const char *html, size_t size) {
char *hts_getCharsetFromMeta(const char *html, size_t size) {
int i;
// <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" >
for(i = 0 ; i < size ; i++) {
if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0 && is_space(html[i + 5]) ) {
for(i = 0; i < size; i++) {
if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
&& is_space(html[i + 5])) {
/* Skip spaces */
for(i += 5 ; is_space(html[i]) ; i++) ;
if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0 && is_space_or_equal(html[i + 10]) ) {
for(i += 10 ; is_space_or_equal_or_quote(html[i]) ; i++) ;
for(i += 5; is_space(html[i]); i++) ;
if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
&& is_space_or_equal(html[i + 10])) {
for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
for(i += 12 ; is_space_or_equal_or_quote(html[i]) ; i++) ;
if (strncasecmp(&html[i], "CONTENT", 7) == 0 && is_space_or_equal(html[i + 7]) ) {
for(i += 7 ; is_space_or_equal_or_quote(html[i]) ; i++) ;
for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
if (strncasecmp(&html[i], "CONTENT", 7) == 0
&& is_space_or_equal(html[i + 7])) {
for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
/* Skip content-type */
for( ; i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'' ; i++) ;
for(;
i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
i++) ;
/* Expect charset attribute here */
if (html[i] == ';') {
for(i++ ; is_space(html[i]) ; i++) ;
for(i++; is_space(html[i]); i++) ;
/* Look for charset */
if (strncasecmp(&html[i], "charset", 7) == 0 && is_space_or_equal(html[i + 7])) {
if (strncasecmp(&html[i], "charset", 7) == 0
&& is_space_or_equal(html[i + 7])) {
int len;
for(i += 7 ; is_space_or_equal(html[i]) || html[i] == '\'' ; i++) ;
for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
i++) ;
/* Charset */
for(len = 0 ; i + len < size && html[i + len] != '"' && html[i + len] != '\'' && html[i + len] != ' ' ; len++) ;
for(len = 0;
i + len < size && html[i + len] != '"'
&& html[i + len] != '\'' && html[i + len] != ' '; len++) ;
/* No error ? */
if (len != 0 && i < size) {
char *const s = malloc(len + 1);
int j;
for(j = 0 ; j < len ; j++) {
for(j = 0; j < len; j++) {
s[j] = html[i + j];
}
s[len] = '\0';
@@ -557,4 +601,3 @@ char* hts_getCharsetFromMeta(const char *html, size_t size) {
}
return NULL;
}

13
src/htscharset.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Charset conversion functions */
/* Author: Xavier Roche */
@@ -51,18 +48,20 @@ Please visit our Website: http://www.httrack.com
* Convert the string "s" from charset "charset" to UTF-8.
* Return NULL upon error.
**/
extern char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset);
extern char *hts_convertStringToUTF8(const char *s, size_t size,
const char *charset);
/**
* Convert the string "s" from UTF-8 to charset "charset".
* Return NULL upon error.
**/
extern char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset);
extern char *hts_convertStringFromUTF8(const char *s, size_t size,
const char *charset);
/**
* Extract the charset from the HTML buffer "html"
**/
extern char* hts_getCharsetFromMeta(const char *html, size_t size);
extern char *hts_getCharsetFromMeta(const char *html, size_t size);
/**
* Get an UTF-8 string length in characters.
@@ -74,7 +73,7 @@ extern size_t hts_stringLengthUTF8(const char *s);
/**
* Convert UTF-8 to WCHAR.
**/
extern LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int* pwsize);
extern LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int *pwsize);
/**
* Convert from WCHAR.

3
src/htsconfig.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Global engine definition file */
/* Author: Xavier Roche */

3049
src/htscore.c
View File

File diff suppressed because it is too large Load Diff

253
src/htscore.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Main file .h */
/* Author: Xavier Roche */
@@ -143,23 +140,23 @@ typedef struct filecreate_params filecreate_params;
typedef struct lien_url lien_url;
#endif
struct lien_url {
char firstblock; // flag 1=premier malloc
char link_import; // lien importé à la suite d'un moved - ne pas appliquer les règles classiques up/down
int depth; // profondeur autorisée lien ; >0 forte 0=faible
int pass2; // traiter après les autres, seconde passe. si == -1, lien traité en background
int premier; // pointeur sur le premier lien qui a donné lieu aux autres liens du domaine
int precedent; // pointeur sur le lien qui a donné lieu à ce lien précis
char firstblock; // flag 1=premier malloc
char link_import; // lien importé à la suite d'un moved - ne pas appliquer les règles classiques up/down
int depth; // profondeur autorisée lien ; >0 forte 0=faible
int pass2; // traiter après les autres, seconde passe. si == -1, lien traité en background
int premier; // pointeur sur le premier lien qui a donné lieu aux autres liens du domaine
int precedent; // pointeur sur le lien qui a donné lieu à ce lien précis
//int moved; // pointeur sur moved
int retry; // nombre de retry restants
int testmode; // mode test uniquement, envoyer juste un head!
char* adr; // adresse
char* fil; // nom du fichier distant
char* sav; // nom à sauver sur disque (avec chemin éventuel)
char* cod; // chemin codebase éventuel si classe java
char* former_adr; // adresse initiale (avant éventuel moved), peut être nulle
char* former_fil; // nom du fichier distant initial (avant éventuel moved), peut être nul
int retry; // nombre de retry restants
int testmode; // mode test uniquement, envoyer juste un head!
char *adr; // adresse
char *fil; // nom du fichier distant
char *sav; // nom à sauver sur disque (avec chemin éventuel)
char *cod; // chemin codebase éventuel si classe java
char *former_adr; // adresse initiale (avant éventuel moved), peut être nulle
char *former_fil; // nom du fichier distant initial (avant éventuel moved), peut être nul
// pour optimisation:
int hash_next[3]; // prochain lien avec même valeur hash
int hash_next[3]; // prochain lien avec même valeur hash
};
// chargement de fichiers en 'arrière plan'
@@ -171,43 +168,43 @@ struct lien_back {
#if DEBUG_CHECKINT
char magic;
#endif
char url_adr[HTS_URLMAXSIZE*2]; // adresse
char url_fil[HTS_URLMAXSIZE*2]; // nom du fichier distant
char url_sav[HTS_URLMAXSIZE*2]; // nom à sauver sur disque (avec chemin éventuel)
char referer_adr[HTS_URLMAXSIZE*2]; // adresse host page referer
char referer_fil[HTS_URLMAXSIZE*2]; // fichier page referer
char location_buffer[HTS_URLMAXSIZE*2]; // "location" en cas de "moved" (302,..)
char* tmpfile; // nom à sauver temporairement (compressé)
char tmpfile_buffer[HTS_URLMAXSIZE*2]; // buffer pour le nom à sauver temporairement
char send_too[1024]; // données à envoyer en même temps que le header
int status; // status (-1=non utilisé, 0: prêt, >0: opération en cours)
int locked; // locked (to be used soon)
int testmode; // mode de test
int timeout; // gérer des timeouts? (!=0 : nombre de secondes)
TStamp timeout_refresh; // si oui, time refresh
int rateout; // timeout refresh? (!=0 : taux minimum toléré en octets/s)
TStamp rateout_time; // si oui, date de départ
LLint maxfile_nonhtml; // taille max d'un fichier non html
LLint maxfile_html; // idem pour un ficheir html
htsblk r; // structure htsblk de chaque objet en background
int is_update; // mode update
int head_request; // requète HEAD?
LLint range_req_size; // range utilisé
TStamp ka_time_start; // refresh time for KA
char url_adr[HTS_URLMAXSIZE * 2]; // adresse
char url_fil[HTS_URLMAXSIZE * 2]; // nom du fichier distant
char url_sav[HTS_URLMAXSIZE * 2]; // nom à sauver sur disque (avec chemin éventuel)
char referer_adr[HTS_URLMAXSIZE * 2]; // adresse host page referer
char referer_fil[HTS_URLMAXSIZE * 2]; // fichier page referer
char location_buffer[HTS_URLMAXSIZE * 2]; // "location" en cas de "moved" (302,..)
char *tmpfile; // nom à sauver temporairement (compressé)
char tmpfile_buffer[HTS_URLMAXSIZE * 2]; // buffer pour le nom à sauver temporairement
char send_too[1024]; // données à envoyer en même temps que le header
int status; // status (-1=non utilisé, 0: prêt, >0: opération en cours)
int locked; // locked (to be used soon)
int testmode; // mode de test
int timeout; // gérer des timeouts? (!=0 : nombre de secondes)
TStamp timeout_refresh; // si oui, time refresh
int rateout; // timeout refresh? (!=0 : taux minimum toléré en octets/s)
TStamp rateout_time; // si oui, date de départ
LLint maxfile_nonhtml; // taille max d'un fichier non html
LLint maxfile_html; // idem pour un ficheir html
htsblk r; // structure htsblk de chaque objet en background
int is_update; // mode update
int head_request; // requète HEAD?
LLint range_req_size; // range utilisé
TStamp ka_time_start; // refresh time for KA
//
int http11; // L'en tête doit être signé HTTP/1.1 et non HTTP/1.0
int is_chunk; // chunk?
char* chunk_adr; // adresse chunk en cours de chargement
LLint chunk_size; // taille chunk en cours de chargement
LLint chunk_blocksize; // taille data declaree par le chunk
LLint compressed_size; // taille compressés (stats uniquement)
int http11; // L'en tête doit être signé HTTP/1.1 et non HTTP/1.0
int is_chunk; // chunk?
char *chunk_adr; // adresse chunk en cours de chargement
LLint chunk_size; // taille chunk en cours de chargement
LLint chunk_blocksize; // taille data declaree par le chunk
LLint compressed_size; // taille compressés (stats uniquement)
//
//int links_index; // to access liens[links_index]
//
char info[256]; // éventuel status pour le ftp
int stop_ftp; // flag stop pour ftp
int finalized; // finalized (optim memory)
int early_add; // was added before link heap saw it
char info[256]; // éventuel status pour le ftp
int stop_ftp; // flag stop pour ftp
int finalized; // finalized (optim memory)
int early_add; // was added before link heap saw it
#if DEBUG_CHECKINT
char magic2;
#endif
@@ -218,10 +215,10 @@ struct lien_back {
typedef struct struct_back struct_back;
#endif
struct struct_back {
lien_back* lnk;
lien_back *lnk;
int count;
inthash ready;
LLint ready_size_bytes;
LLint ready_size_bytes;
};
typedef struct cache_back_zip_entry cache_back_zip_entry;
@@ -232,29 +229,29 @@ typedef struct cache_back_zip_entry cache_back_zip_entry;
typedef struct cache_back cache_back;
#endif
struct cache_back {
int version; // 0 ou 1
int version; // 0 ou 1
/* */
int type;
int ro;
FILE *dat,*ndx,*olddat;
char *use; // liste des adr+fil
FILE *lst; // liste des fichiers pour la "purge"
FILE *txt; // liste des fichiers (info)
FILE *dat, *ndx, *olddat;
char *use; // liste des adr+fil
FILE *lst; // liste des fichiers pour la "purge"
FILE *txt; // liste des fichiers (info)
char lastmodified[256];
// HASH
inthash hashtable;
// HASH for tests (naming subsystem)
inthash cached_tests;
// fichiers log optionnels
FILE* log;
FILE* errlog;
FILE *log;
FILE *errlog;
// variables
int ptr_ant; // pointeur pour anticiper
int ptr_last; // pointeur pour anticiper
int ptr_ant; // pointeur pour anticiper
int ptr_last; // pointeur pour anticiper
//
void *zipInput;
void *zipOutput;
cache_back_zip_entry* zipEntries;
cache_back_zip_entry *zipEntries;
int zipEntriesOffs;
int zipEntriesCapa;
};
@@ -264,9 +261,9 @@ struct cache_back {
typedef struct hash_struct hash_struct;
#endif
struct hash_struct {
lien_url** liens; // pointeur sur liens
int max_lien; // indice le plus grand rencontré
int hash[3][HTS_HASH_SIZE]; // tables pour sav/adr-fil/former_adr-former_fil
lien_url **liens; // pointeur sur liens
int max_lien; // indice le plus grand rencontré
int hash[3][HTS_HASH_SIZE]; // tables pour sav/adr-fil/former_adr-former_fil
};
#ifndef HTS_DEF_FWSTRUCT_filecreate_params
@@ -274,19 +271,19 @@ struct hash_struct {
typedef struct filecreate_params filecreate_params;
#endif
struct filecreate_params {
FILE* lst;
char path[HTS_URLMAXSIZE*2];
FILE *lst;
char path[HTS_URLMAXSIZE * 2];
};
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
HTS_STATIC int cache_writable(cache_back* cache) {
return (cache != NULL && ( cache->dat != NULL || cache->zipOutput != NULL ) );
HTS_STATIC int cache_writable(cache_back * cache) {
return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
}
HTS_STATIC int cache_readable(cache_back* cache) {
return (cache != NULL && ( cache->olddat != NULL || cache->zipInput != NULL ) );
HTS_STATIC int cache_readable(cache_back * cache) {
return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
}
#endif
@@ -300,97 +297,109 @@ HTS_STATIC int cache_readable(cache_back* cache) {
//
#ifndef HTTRACK_DEFLIB
HTSEXT_API int hts_is_parsing(httrackp *opt, int flag);
HTSEXT_API int hts_is_testing(httrackp *opt);
HTSEXT_API int hts_addurl(httrackp *opt, char** url);
HTSEXT_API int hts_resetaddurl(httrackp *opt);
HTSEXT_API int copy_htsopt(const httrackp* from,httrackp* to);
HTSEXT_API char* hts_errmsg(httrackp *opt);
HTSEXT_API int hts_setpause(httrackp *opt, int);
HTSEXT_API int hts_is_parsing(httrackp * opt, int flag);
HTSEXT_API int hts_is_testing(httrackp * opt);
HTSEXT_API int hts_addurl(httrackp * opt, char **url);
HTSEXT_API int hts_resetaddurl(httrackp * opt);
HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to);
HTSEXT_API char *hts_errmsg(httrackp * opt);
HTSEXT_API int hts_setpause(httrackp * opt, int);
//
HTSEXT_API int hts_is_exiting(httrackp *opt);
HTSEXT_API int hts_request_stop(httrackp* opt, int force);
HTSEXT_API int hts_is_exiting(httrackp * opt);
HTSEXT_API int hts_request_stop(httrackp * opt, int force);
//
HTSEXT_API int hts_cancel_file_push(httrackp *opt, const char *url);
HTSEXT_API void hts_cancel_test(httrackp *opt);
HTSEXT_API void hts_cancel_parsing(httrackp *opt);
HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url);
HTSEXT_API void hts_cancel_test(httrackp * opt);
HTSEXT_API void hts_cancel_parsing(httrackp * opt);
#endif
char* hts_cancel_file_pop(httrackp *opt);
char *hts_cancel_file_pop(httrackp * opt);
#endif
//
//int httpmirror(char* url,int level,httrackp opt);
int httpmirror(char* url1,httrackp* opt);
int filesave(httrackp* opt,const char* adr,int len,const char* s,const char* url_adr /* = NULL */,const char* url_fil /* = NULL */);
char* hts_cancel_file_pop(httrackp *opt);
int httpmirror(char *url1, httrackp * opt);
int filesave(httrackp * opt, const char *adr, int len, const char *s,
const char *url_adr /* = NULL */ ,
const char *url_fil /* = NULL */ );
char *hts_cancel_file_pop(httrackp * opt);
int check_fatal_io_errno(void);
int engine_stats(void);
void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,struct_back* sback,char* host);
FILE* filecreate(filenote_strc *strct,const char* s);
FILE* fileappend(filenote_strc *strct,const char* s);
int filecreateempty(filenote_strc *strct, const char* filename);
int filenote(filenote_strc *strct,const char* s,filecreate_params* params);
void file_notify(httrackp* opt,const char* adr,const char* fil,const char* save,int create,int modify,int wasupdated);
void usercommand(httrackp* opt,int exe,const char* cmd,const char* file,const char* adr,const char* fil);
void usercommand_exe(const char* cmd,const char* file);
int filters_init(char*** ptrfilters, int maxfilter, int filterinc);
void host_ban(httrackp * opt, lien_url ** liens, int ptr, int lien_tot,
struct_back * sback, char *host);
FILE *filecreate(filenote_strc * strct, const char *s);
FILE *fileappend(filenote_strc * strct, const char *s);
int filecreateempty(filenote_strc * strct, const char *filename);
int filenote(filenote_strc * strct, const char *s, filecreate_params * params);
void file_notify(httrackp * opt, const char *adr, const char *fil,
const char *save, int create, int modify, int wasupdated);
void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
const char *adr, const char *fil);
void usercommand_exe(const char *cmd, const char *file);
int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
#ifndef HTTRACK_DEFLIB
HTSEXT_API int structcheck(const char* path);
HTSEXT_API int structcheck_utf8(const char* path);
HTSEXT_API int dir_exists(const char* path);
HTSEXT_API int structcheck(const char *path);
HTSEXT_API int structcheck_utf8(const char *path);
HTSEXT_API int dir_exists(const char *path);
#endif
int fspc(httrackp *opt,FILE* fp,const char* type);
char* next_token(char* p,int flag);
int fspc(httrackp * opt, FILE * fp, const char *type);
char *next_token(char *p, int flag);
//
char* readfile(char* fil);
char* readfile2(char* fil, LLint* size);
char* readfile_utf8(char* fil);
char* readfile_or(char* fil,char* defaultdata);
char *readfile(char *fil);
char *readfile2(char *fil, LLint * size);
char *readfile_utf8(char *fil);
char *readfile_or(char *fil, char *defaultdata);
#if 0
void check_rate(TStamp stat_timestart,int maxrate);
void check_rate(TStamp stat_timestart, int maxrate);
#endif
// liens
int liens_record(char* adr,char* fil,char* save,char* former_adr,char* former_fil,char* codebase);
int liens_record(char *adr, char *fil, char *save, char *former_adr,
char *former_fil, char *codebase);
// backing, routines externes
int back_pluggable_sockets(struct_back* sback, httrackp* opt);
int back_pluggable_sockets_strict(struct_back* sback, httrackp* opt);
int back_fill(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot);
int backlinks_done(struct_back* sback,lien_url** liens,int lien_tot,int ptr);
int back_fillmax(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot);
int back_pluggable_sockets(struct_back * sback, httrackp * opt);
int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
lien_url ** liens, int ptr, int numero_passe, int lien_tot);
int backlinks_done(struct_back * sback, lien_url ** liens, int lien_tot,
int ptr);
int back_fillmax(struct_back * sback, httrackp * opt, cache_back * cache,
lien_url ** liens, int ptr, int numero_passe, int lien_tot);
// cancel file
#ifndef HTTRACK_DEFLIB
HTSEXT_API int hts_cancel_file_push(httrackp *opt, const char *url);
HTSEXT_API void hts_cancel_test(httrackp *opt);
HTSEXT_API void hts_cancel_parsing(httrackp *opt);
HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url);
HTSEXT_API void hts_cancel_test(httrackp * opt);
HTSEXT_API void hts_cancel_parsing(httrackp * opt);
#endif
int ask_continue(httrackp *opt);
int ask_continue(httrackp * opt);
int nombre_digit(int n);
// Java
#if 0
int hts_add_file(char* file,int file_position);
int hts_add_file(char *file, int file_position);
#endif
// Polling
#if HTS_POLL
int check_flot(T_SOC s);
int check_stdin(void);
int read_stdin(char* s,int max);
int read_stdin(char *s, int max);
#endif
int check_sockerror(T_SOC s);
int check_sockdata(T_SOC s);
/* external modules */
int htsAddLink(htsmoduleStruct* str, char* link);
int htsAddLink(htsmoduleStruct * str, char *link);
// Void
void voidf(void);

3654
src/htscoremain.c
View File

File diff suppressed because it is too large Load Diff

14
src/htscoremain.h
View File

@@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* main routine (first called) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSMAINHSR_DEFH
#define HTSMAINHSR_DEFH
#define HTSMAINHSR_DEFH
// --assume standard
#define HTS_ASSUME_STANDARD \
@@ -52,13 +47,12 @@ Please visit our Website: http://www.httrack.com
// Main, récupère les paramètres et appelle le robot
#ifndef HTTRACK_DEFLIB
HTSEXT_API int hts_main(int argc, char **argv);
HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt);
HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt);
#endif
int cmdl_opt(char* s);
int check_path(String* s,char* defaultname);
int cmdl_opt(char *s);
int check_path(String * s, char *defaultname);
#endif
#endif

190
src/htsdefines.h
View File

@@ -23,11 +23,9 @@ Important notes:
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Some defines for httrack.c and others */
/* Author: Xavier Roche */
@@ -72,71 +70,94 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
#ifdef _WIN32
#define EXTERNAL_FUNCTION __declspec(dllexport)
#else
#define EXTERNAL_FUNCTION
#define EXTERNAL_FUNCTION
#endif
#endif
/* --wrapper plug function prototype */
typedef int (*t_hts_plug)(httrackp *opt, const char* argv);
typedef int (*t_hts_unplug)(httrackp *opt);
typedef int (*t_hts_plug) (httrackp * opt, const char *argv);
typedef int (*t_hts_unplug) (httrackp * opt);
/* htsopt function callbacks definitions */
typedef void (* t_hts_htmlcheck_init)(t_hts_callbackarg *carg);
typedef void (* t_hts_htmlcheck_uninit)(t_hts_callbackarg *carg);
typedef int (* t_hts_htmlcheck_start)(t_hts_callbackarg *carg, httrackp* opt);
typedef int (* t_hts_htmlcheck_end)(t_hts_callbackarg *carg, httrackp *opt);
typedef int (* t_hts_htmlcheck_chopt)(t_hts_callbackarg *carg, httrackp* opt);
typedef int (* t_hts_htmlcheck_process)(t_hts_callbackarg *carg, httrackp *opt,
char** html, int* len, const char* url_adresse, const char* url_fichier);
typedef void (*t_hts_htmlcheck_init) (t_hts_callbackarg * carg);
typedef void (*t_hts_htmlcheck_uninit) (t_hts_callbackarg * carg);
typedef int (*t_hts_htmlcheck_start) (t_hts_callbackarg * carg, httrackp * opt);
typedef int (*t_hts_htmlcheck_end) (t_hts_callbackarg * carg, httrackp * opt);
typedef int (*t_hts_htmlcheck_chopt) (t_hts_callbackarg * carg, httrackp * opt);
typedef int (*t_hts_htmlcheck_process) (t_hts_callbackarg * carg,
httrackp * opt, char **html, int *len,
const char *url_adresse,
const char *url_fichier);
typedef t_hts_htmlcheck_process t_hts_htmlcheck_preprocess;
typedef t_hts_htmlcheck_process t_hts_htmlcheck_postprocess;
typedef int (* t_hts_htmlcheck_check_html)(t_hts_callbackarg *carg, httrackp *opt,
char* html, int len, const char* url_adresse, const char* url_fichier);
typedef const char* (* t_hts_htmlcheck_query)(t_hts_callbackarg *carg, httrackp *opt,
const char* question);
typedef const char* (* t_hts_htmlcheck_query2)(t_hts_callbackarg *carg, httrackp *opt,
const char* question);
typedef const char* (* t_hts_htmlcheck_query3)(t_hts_callbackarg *carg, httrackp *opt,
const char* question);
typedef int (* t_hts_htmlcheck_loop)(t_hts_callbackarg *carg, httrackp *opt,
lien_back* back, int back_max, int back_index,
int lien_tot, int lien_ntot,
int stat_time, hts_stat_struct* stats);
typedef int (* t_hts_htmlcheck_check_link)(t_hts_callbackarg *carg, httrackp *opt,
const char* adr, const char* fil, int status);
typedef int (* t_hts_htmlcheck_check_mime)(t_hts_callbackarg *carg, httrackp *opt,
const char* adr, const char* fil, const char* mime, int status);
typedef void (* t_hts_htmlcheck_pause)(t_hts_callbackarg *carg, httrackp *opt,
const char* lockfile);
typedef void (* t_hts_htmlcheck_filesave)(t_hts_callbackarg *carg, httrackp *opt,
const char* file);
typedef void (* t_hts_htmlcheck_filesave2)(t_hts_callbackarg *carg, httrackp *opt,
const char* hostname, const char* filename, const char* localfile,
int is_new, int is_modified, int not_updated);
typedef int (* t_hts_htmlcheck_linkdetected)(t_hts_callbackarg *carg, httrackp *opt,
char* link);
typedef int (* t_hts_htmlcheck_linkdetected2)(t_hts_callbackarg *carg, httrackp *opt,
char* link, const char* tag_start);
typedef int (* t_hts_htmlcheck_xfrstatus)(t_hts_callbackarg *carg, httrackp *opt,
lien_back* back);
typedef int (* t_hts_htmlcheck_savename)(t_hts_callbackarg *carg, httrackp *opt,
const char* adr_complete, const char* fil_complete,
const char* referer_adr, const char* referer_fil, char* save);
typedef int (*t_hts_htmlcheck_check_html) (t_hts_callbackarg * carg,
httrackp * opt, char *html, int len,
const char *url_adresse,
const char *url_fichier);
typedef const char *(*t_hts_htmlcheck_query) (t_hts_callbackarg * carg,
httrackp * opt,
const char *question);
typedef const char *(*t_hts_htmlcheck_query2) (t_hts_callbackarg * carg,
httrackp * opt,
const char *question);
typedef const char *(*t_hts_htmlcheck_query3) (t_hts_callbackarg * carg,
httrackp * opt,
const char *question);
typedef int (*t_hts_htmlcheck_loop) (t_hts_callbackarg * carg, httrackp * opt,
lien_back * back, int back_max,
int back_index, int lien_tot,
int lien_ntot, int stat_time,
hts_stat_struct * stats);
typedef int (*t_hts_htmlcheck_check_link) (t_hts_callbackarg * carg,
httrackp * opt, const char *adr,
const char *fil, int status);
typedef int (*t_hts_htmlcheck_check_mime) (t_hts_callbackarg * carg,
httrackp * opt, const char *adr,
const char *fil, const char *mime,
int status);
typedef void (*t_hts_htmlcheck_pause) (t_hts_callbackarg * carg, httrackp * opt,
const char *lockfile);
typedef void (*t_hts_htmlcheck_filesave) (t_hts_callbackarg * carg,
httrackp * opt, const char *file);
typedef void (*t_hts_htmlcheck_filesave2) (t_hts_callbackarg * carg,
httrackp * opt, const char *hostname,
const char *filename,
const char *localfile, int is_new,
int is_modified, int not_updated);
typedef int (*t_hts_htmlcheck_linkdetected) (t_hts_callbackarg * carg,
httrackp * opt, char *link);
typedef int (*t_hts_htmlcheck_linkdetected2) (t_hts_callbackarg * carg,
httrackp * opt, char *link,
const char *tag_start);
typedef int (*t_hts_htmlcheck_xfrstatus) (t_hts_callbackarg * carg,
httrackp * opt, lien_back * back);
typedef int (*t_hts_htmlcheck_savename) (t_hts_callbackarg * carg,
httrackp * opt,
const char *adr_complete,
const char *fil_complete,
const char *referer_adr,
const char *referer_fil, char *save);
typedef t_hts_htmlcheck_savename t_hts_htmlcheck_extsavename;
typedef int (* t_hts_htmlcheck_sendhead)(t_hts_callbackarg *carg, httrackp *opt,
char* buff, const char* adr, const char* fil,
const char* referer_adr, const char* referer_fil,
htsblk* outgoing);
typedef int (* t_hts_htmlcheck_receivehead)(t_hts_callbackarg *carg, httrackp *opt,
char* buff, const char* adr, const char* fil,
const char* referer_adr, const char* referer_fil,
htsblk* incoming);
typedef int (*t_hts_htmlcheck_sendhead) (t_hts_callbackarg * carg,
httrackp * opt, char *buff,
const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil,
htsblk * outgoing);
typedef int (*t_hts_htmlcheck_receivehead) (t_hts_callbackarg * carg,
httrackp * opt, char *buff,
const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil,
htsblk * incoming);
/* External additional parsing module(s) */
typedef int (*t_hts_htmlcheck_detect)(t_hts_callbackarg *carg, httrackp *opt, htsmoduleStruct* str);
typedef int (*t_hts_htmlcheck_parse)(t_hts_callbackarg *carg, httrackp *opt, htsmoduleStruct* str);
typedef int (*t_hts_htmlcheck_detect) (t_hts_callbackarg * carg, httrackp * opt,
htsmoduleStruct * str);
typedef int (*t_hts_htmlcheck_parse) (t_hts_callbackarg * carg, httrackp * opt,
htsmoduleStruct * str);
/* Callbacks */
#ifndef HTS_DEF_FWSTRUCT_t_hts_htmlcheck_callbacks
@@ -152,7 +173,7 @@ typedef struct t_hts_htmlcheck_callbacks t_hts_htmlcheck_callbacks;
} NAME
/* Callback items */
typedef void* t_hts_htmlcheck_t_hts_htmlcheck_callbacks_item;
typedef void *t_hts_htmlcheck_t_hts_htmlcheck_callbacks_item;
typedef DEFCALLBACK(t_hts_htmlcheck_callbacks_item);
/* Linked list, which should be used for the 'arg' user-defined argument */
@@ -170,33 +191,33 @@ struct t_hts_callbackarg {
/* Callback structure */
struct t_hts_htmlcheck_callbacks {
/* v3.41 */
DEFCALLBACK(init);
DEFCALLBACK(uninit);
DEFCALLBACK(start);
DEFCALLBACK(end);
DEFCALLBACK(chopt);
DEFCALLBACK(preprocess);
DEFCALLBACK(postprocess);
DEFCALLBACK(check_html);
DEFCALLBACK(query);
DEFCALLBACK(query2);
DEFCALLBACK(query3);
DEFCALLBACK(loop);
DEFCALLBACK(check_link);
DEFCALLBACK(check_mime);
DEFCALLBACK(pause);
DEFCALLBACK(filesave);
DEFCALLBACK(filesave2);
DEFCALLBACK(linkdetected);
DEFCALLBACK(linkdetected2);
DEFCALLBACK(xfrstatus);
DEFCALLBACK(savename);
DEFCALLBACK(sendhead);
DEFCALLBACK(receivehead);
DEFCALLBACK(init);
DEFCALLBACK(uninit);
DEFCALLBACK(start);
DEFCALLBACK(end);
DEFCALLBACK(chopt);
DEFCALLBACK(preprocess);
DEFCALLBACK(postprocess);
DEFCALLBACK(check_html);
DEFCALLBACK(query);
DEFCALLBACK(query2);
DEFCALLBACK(query3);
DEFCALLBACK(loop);
DEFCALLBACK(check_link);
DEFCALLBACK(check_mime);
DEFCALLBACK(pause);
DEFCALLBACK(filesave);
DEFCALLBACK(filesave2);
DEFCALLBACK(linkdetected);
DEFCALLBACK(linkdetected2);
DEFCALLBACK(xfrstatus);
DEFCALLBACK(savename);
DEFCALLBACK(sendhead);
DEFCALLBACK(receivehead);
DEFCALLBACK(detect);
DEFCALLBACK(parse);
/* >3.41 */
DEFCALLBACK(extsavename);
DEFCALLBACK(extsavename);
};
/* Library internal definitions */
@@ -207,8 +228,8 @@ struct t_hts_htmlcheck_callbacks {
typedef struct t_hts_callback_ref t_hts_callback_ref;
#endif
struct t_hts_callback_ref {
const char *name;
size_t offset;
const char *name;
size_t offset;
};
extern const t_hts_htmlcheck_callbacks default_callbacks;
@@ -216,14 +237,13 @@ extern const t_hts_callback_ref default_callbacks_ref[];
#define HT_PRINT(A) strcatbuff(opt->state.HTbuff,A);
#define HT_REQUEST_START opt->state.HTbuff[0]='\0';
#define HT_REQUEST_END
#define HT_REQUEST_END
#define HTT_REQUEST_START opt->state.HTbuff[0]='\0';
#define HTT_REQUEST_END
#define HTT_REQUEST_END
#define HTS_REQUEST_START opt->state.HTbuff[0]='\0';
#define HTS_REQUEST_END
#define HTS_REQUEST_END
#define HTS_PANIC_PRINTF(S) strcpybuff(opt->state._hts_errmsg,S);
#endif
#endif

292
src/htsfilters.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* filters ("regexp") */
@@ -38,7 +35,6 @@ Please visit our Website: http://www.httrack.com
/* Internal engine bytecode */
#define HTS_INTERNAL_BYTECODE
// *.gif match all gif files
// *[file]/*[file].exe match all exe files with one folder structure
// *[A-Z,a-z,0-9,/,?] match letters, nums, / and ?
@@ -59,43 +55,45 @@ Please visit our Website: http://www.httrack.com
// optionnel: taille à contrôller (ou numéro, etc) en pointeur
// (en de détection de *size, la taille limite est écrite par dessus *size)
// exemple: +-*.gif*[<5] == supprimer GIF si <5KB
int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth) {
int verdict = 0; // on sait pas
int fa_strjoker(int type, char **filters, int nfil, char *nom, LLint * size,
int *size_flag, int *depth) {
int verdict = 0; // on sait pas
int i;
LLint sizelimit=0;
LLint sizelimit = 0;
if (size)
sizelimit=*size;
for(i=0;i<nfil;i++) {
sizelimit = *size;
for(i = 0; i < nfil; i++) {
LLint sz;
int filteroffs = 1;
if (strncmp(filters[i] + filteroffs, "mime:", 5) == 0) {
if (type == 0) // regular filters
if (type == 0) // regular filters
continue;
filteroffs += 5; // +mime:text/html
} else { // mime filters
filteroffs += 5; // +mime:text/html
} else { // mime filters
if (type != 0)
continue;
}
if (size)
sz=*size;
if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) { // reconnu
sz = *size;
if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) { // reconnu
if (size)
if (sz != *size)
sizelimit=sz;
if (filters[i][0]=='+')
verdict = 1; // autorisé
if (sz != *size)
sizelimit = sz;
if (filters[i][0] == '+')
verdict = 1; // autorisé
else
verdict = -1; // interdit
verdict = -1; // interdit
if (depth)
*depth=i;
*depth = i;
}
}
if (size)
*size=sizelimit;
*size = sizelimit;
return verdict;
}
// supercomparateur joker (tm)
// compare a et b (b=avec joker dedans), case insensitive [voir CI]
// renvoi l'adresse de la première lettre de la chaine
@@ -104,211 +102,232 @@ int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size
// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
// * = toute lettre
// --?-- : spécifique à HTTrack et aux ?
HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag) {
HTS_INLINE char *strjoker(char *chaine, char *joker, LLint * size,
int *size_flag) {
//int err=0;
if (strnotempty(joker)==0) { // fin de chaine joker
if (strnotempty(chaine)==0) // fin aussi pour la chaine: ok
if (strnotempty(joker) == 0) { // fin de chaine joker
if (strnotempty(chaine) == 0) // fin aussi pour la chaine: ok
return chaine;
else if (chaine[0]=='?')
return chaine; // --?-- pour les index.html?Choix=2
else if (chaine[0] == '?')
return chaine; // --?-- pour les index.html?Choix=2
else
return NULL; // non trouvé
return NULL; // non trouvé
}
// on va progresser en suivant les 'mots' contenus dans le joker
// un mot peut être un * ou bien toute autre séquence de lettres
if (strcmp(joker,"*")==0) { // ok, rien après
if (strcmp(joker, "*") == 0) { // ok, rien après
return chaine;
}
// 1er cas: jokers * ou jokers multiples *[..]
if (joker[0]=='*') { // comparer joker+reste (*toto/..)
int jmp; // nombre de caractères pour le prochain mot dans joker
int cut = 0; // interdire tout caractère superflu
if (joker[0] == '*') { // comparer joker+reste (*toto/..)
int jmp; // nombre de caractères pour le prochain mot dans joker
int cut = 0; // interdire tout caractère superflu
char pass[256];
char LEFT='[',RIGHT=']';
int unique=0;
char LEFT = '[', RIGHT = ']';
int unique = 0;
switch(joker[1]) {
case '[':
LEFT='[';
RIGHT=']';
unique=0;
switch (joker[1]) {
case '[':
LEFT = '[';
RIGHT = ']';
unique = 0;
break;
case '(':
LEFT='(';
RIGHT=')';
unique=1;
case '(':
LEFT = '(';
RIGHT = ')';
unique = 1;
break;
}
if ((joker[1]==LEFT) && (joker[2]!=LEFT)) { // multijoker (tm)
if ((joker[1] == LEFT) && (joker[2] != LEFT)) { // multijoker (tm)
int i;
for(i=0;i<256;i++) pass[i]=0;
for(i = 0; i < 256; i++)
pass[i] = 0;
// noms réservés
if ((strfield(joker+2,"file")) || (strfield(joker+2,"name"))) {
for(i=0;i<256;i++) pass[i]=1;
if ((strfield(joker + 2, "file")) || (strfield(joker + 2, "name"))) {
for(i = 0; i < 256; i++)
pass[i] = 1;
pass[(int) '?'] = 0;
//pass[(int) ';'] = 0;
pass[(int) '/'] = 0;
i=2;
{ int len=(int) strlen(joker);
while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) i++;
i = 2;
{
int len = (int) strlen(joker);
while((joker[i] != RIGHT) && (joker[i]) && (i < len))
i++;
}
} else if (strfield(joker+2,"path")) {
for(i=0;i<256;i++) pass[i]=1;
} else if (strfield(joker + 2, "path")) {
for(i = 0; i < 256; i++)
pass[i] = 1;
pass[(int) '?'] = 0;
//pass[(int) ';'] = 0;
i=2;
{ int len=(int) strlen(joker);
while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) i++;
i = 2;
{
int len = (int) strlen(joker);
while((joker[i] != RIGHT) && (joker[i]) && (i < len))
i++;
}
} else if (strfield(joker+2,"param")) {
if (chaine[0]=='?') { // il y a un paramètre juste là
for(i=0;i<256;i++) pass[i]=1;
} // sinon synonyme de 'rien'
i=2;
{ int len=(int) strlen(joker);
while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) i++;
} else if (strfield(joker + 2, "param")) {
if (chaine[0] == '?') { // il y a un paramètre juste là
for(i = 0; i < 256; i++)
pass[i] = 1;
} // sinon synonyme de 'rien'
i = 2;
{
int len = (int) strlen(joker);
while((joker[i] != RIGHT) && (joker[i]) && (i < len))
i++;
}
} else {
// décode les directives comme *[A-Z,âêîôû,0-9]
i=2;
if (joker[i] == RIGHT) { // *[] signifie "plus rien après"
cut = 1; // caractère supplémentaire interdit
i = 2;
if (joker[i] == RIGHT) { // *[] signifie "plus rien après"
cut = 1; // caractère supplémentaire interdit
} else {
int len=(int) strlen(joker);
while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) {
if ( (joker[i]=='<') || (joker[i]=='>') ) { // *[<10]
int lsize=0;
int len = (int) strlen(joker);
while((joker[i] != RIGHT) && (joker[i]) && (i < len)) {
if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10]
int lsize = 0;
int lverdict;
i++;
if (sscanf(joker+i,"%d",&lsize) == 1) {
if (sscanf(joker + i, "%d", &lsize) == 1) {
if (size) {
if (*size>=0) {
if (*size >= 0) {
if (size_flag)
*size_flag=1; /* a joué */
if (joker[i-1]=='<')
lverdict=(*size<lsize);
*size_flag = 1; /* a joué */
if (joker[i - 1] == '<')
lverdict = (*size < lsize);
else
lverdict=(*size>lsize);
lverdict = (*size > lsize);
if (!lverdict) {
return NULL; // ne correspond pas
return NULL; // ne correspond pas
} else {
*size=lsize;
return chaine; // ok
*size = lsize;
return chaine; // ok
}
} else
return NULL; // ne correspond pas
return NULL; // ne correspond pas
} else
return NULL; // ne correspond pas (test impossible)
return NULL; // ne correspond pas (test impossible)
// jump
while(isdigit((unsigned char)joker[i])) i++;
while(isdigit((unsigned char) joker[i]))
i++;
}
}
else if (joker[i+1]=='-') { // 2 car, ex: *[A-Z]
if ((int) (unsigned char) joker[i+2]>(int) (unsigned char) joker[i]) {
} else if (joker[i + 1] == '-') { // 2 car, ex: *[A-Z]
if ((int) (unsigned char) joker[i + 2] >
(int) (unsigned char) joker[i]) {
int j;
for(j=(int) (unsigned char) joker[i];j<=(int) (unsigned char) joker[i+2];j++)
pass[j]=1;
for(j = (int) (unsigned char) joker[i];
j <= (int) (unsigned char) joker[i + 2]; j++)
pass[j] = 1;
}
// else err=1;
i+=3;
i += 3;
} else { // 1 car, ex: *[ ]
if (joker[i+2]=='\\' && joker[i+3] != 0) { // escaped char, such as *[\[] or *[\]]
if (joker[i + 2] == '\\' && joker[i + 3] != 0) { // escaped char, such as *[\[] or *[\]]
i++;
}
pass[(int) (unsigned char) joker[i]]=1;
pass[(int) (unsigned char) joker[i]] = 1;
i++;
}
if ((joker[i]==',') || (joker[i]==';')) i++;
if ((joker[i] == ',') || (joker[i] == ';'))
i++;
}
}
}
// à sauter dans joker
jmp=i;
if (joker[i]) jmp++;
jmp = i;
if (joker[i])
jmp++;
//
} else { // tout autoriser
} else { // tout autoriser
//
int i;
for(i=0;i<256;i++) pass[i]=1; // tout autoriser
jmp=1;
for(i = 0; i < 256; i++)
pass[i] = 1; // tout autoriser
jmp = 1;
////if (joker[2]==LEFT) jmp=3; // permet de recher *<crochet ouvrant>
}
{
int i,max;
char* adr;
int i, max;
char *adr;
// la chaine doit se terminer exactement
if (cut) {
if (strnotempty(chaine))
return NULL; // perdu
return NULL; // perdu
else
return chaine; // ok
return chaine; // ok
}
// comparaison en boucle, c'est ca qui consomme huhu..
// le tableau pass[256] indique les caractères ASCII autorisés
// tester sans le joker (pas ()+ mais ()*)
if (!unique) {
if ( (adr=strjoker(chaine,joker+jmp,size,size_flag)) ) {
if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
return adr;
}
}
// tester
i=0;
i = 0;
if (!unique)
max = (int) strlen(chaine);
else /* *(a) only match a (not aaaaa) */
max=1;
while(i<(int) max) {
if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé
if ( (adr=strjoker(chaine+i+1,joker+jmp,size,size_flag)) ) {
else /* *(a) only match a (not aaaaa) */
max = 1;
while(i < (int) max) {
if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé
if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
return adr;
}
i++;
} else i=max+2; // sortir
} else
i = max + 2; // sortir
}
// tester chaîne vide
if (i!=max+2) // avant c'est ok
if ( (adr=strjoker(chaine+max,joker+jmp,size,size_flag)) )
return adr;
return NULL; // perdu
if (i != max + 2) // avant c'est ok
if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
return adr;
return NULL; // perdu
}
} else { // comparer mot+reste (toto*..)
} else { // comparer mot+reste (toto*..)
if (strnotempty(chaine)) {
int jmp=0,ok=1;
int jmp = 0, ok = 1;
// comparer début de joker et début de chaine
while((joker[jmp]!='*') && (joker[jmp]) && (ok)) {
while((joker[jmp] != '*') && (joker[jmp]) && (ok)) {
// CI : remplacer streql par une comparaison !=
if (!streql(chaine[jmp],joker[jmp])) {
ok=0; // quitter
if (!streql(chaine[jmp], joker[jmp])) {
ok = 0; // quitter
}
jmp++;
}
// comparaison ok?
if (ok) {
// continuer la comparaison.
if (strjoker(chaine+jmp,joker+jmp,size,size_flag))
return chaine; // retourner 1e lettre
if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
return chaine; // retourner 1e lettre
}
} // strlen(a)
return NULL;
} // * ou mot
} // strlen(a)
return NULL;
} // * ou mot
return NULL;
}
@@ -317,10 +336,11 @@ HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag) {
// exemple: find dans un texte de strcpybuff(*[A-Z,a-z],"*[0-9]"); va rechercher la première occurence
// d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
// ATTENTION!! Eviter les jokers en début, où gare au temps machine!
char* strjokerfind(char* chaine,char* joker) {
char* adr;
char *strjokerfind(char *chaine, char *joker) {
char *adr;
while(*chaine) {
if ( (adr=strjoker(chaine,joker,NULL,NULL)) ) { // ok trouvé
if ((adr = strjoker(chaine, joker, NULL, NULL))) { // ok trouvé
return adr;
}
chaine++;

15
src/htsfilters.h
View File

@@ -17,37 +17,34 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* filters ("regexp") */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSFILT_DEFH
#define HTSFILT_DEFH
#define HTSFILT_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
#include "htsbase.h"
int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth);
HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag);
char* strjokerfind(char* chaine,char* joker);
int fa_strjoker(int type, char **filters, int nfil, char *nom, LLint * size,
int *size_flag, int *depth);
HTS_INLINE char *strjoker(char *chaine, char *joker, LLint * size,
int *size_flag);
char *strjokerfind(char *chaine, char *joker);
#endif
#endif

1073
src/htsftp.c
View File

File diff suppressed because it is too large Load Diff

33
src/htsftp.h
View File

@@ -17,26 +17,22 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: basic FTP protocol manager .h */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSFTP_DEFH
#define HTSFTP_DEFH
#define HTSFTP_DEFH
#include "htsbase.h"
#include "htsbasenet.h"
@@ -58,30 +54,29 @@ typedef struct httrackp httrackp;
typedef struct FTPDownloadStruct FTPDownloadStruct;
#endif
struct FTPDownloadStruct {
lien_back *pBack;
httrackp *pOpt;
lien_back *pBack;
httrackp *pOpt;
};
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
#if USE_BEGINTHREAD
void launch_ftp(FTPDownloadStruct *params);
void back_launch_ftp( void* pP );
void launch_ftp(FTPDownloadStruct * params);
void back_launch_ftp(void *pP);
#else
void launch_ftp(FTPDownloadStruct *params,char* path,char* exec);
int back_launch_ftp(FTPDownloadStruct *params);
void launch_ftp(FTPDownloadStruct * params, char *path, char *exec);
int back_launch_ftp(FTPDownloadStruct * params);
#endif
int run_launch_ftp(FTPDownloadStruct *params);
int send_line(T_SOC soc,char* data);
int get_ftp_line(T_SOC soc,char* line,int timeout);
T_SOC get_datasocket(char* to_send);
int stop_ftp(lien_back* back);
char* linejmp(char* line);
int run_launch_ftp(FTPDownloadStruct * params);
int send_line(T_SOC soc, char *data);
int get_ftp_line(T_SOC soc, char *line, int timeout);
T_SOC get_datasocket(char *to_send);
int stop_ftp(lien_back * back);
char *linejmp(char *line);
int check_socket(T_SOC soc);
int check_socket_connect(T_SOC soc);
int wait_socket_receive(T_SOC soc,int timeout);
int wait_socket_receive(T_SOC soc, int timeout);
#endif
#endif

66
src/htsglobal.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Global #define file */
/* Author: Xavier Roche */
@@ -71,7 +68,6 @@ Please visit our Website: http://www.httrack.com
#endif
#endif
// config.h
#ifdef _WIN32
@@ -101,7 +97,7 @@ Please visit our Website: http://www.httrack.com
#define HTS_SPARE_MEMORY 1
#define HTS_ALIGN 8
#define BIGSTK static
#undef DLLIB // LoadLibrary(libssl) crashes
#undef DLLIB // LoadLibrary(libssl) crashes
#define NOSTRDEBUG 1
#undef HTS_MAKE_KEYWORD_INDEX
#ifdef HTS_CECOMPAT
@@ -175,7 +171,7 @@ Please visit our Website: http://www.httrack.com
#define HTS_SPARE_MEMORY 0
#endif
#ifndef BIGSTK
#ifndef BIGSTK
#define BIGSTK
#endif
@@ -304,7 +300,7 @@ Please visit our Website: http://www.httrack.com
#define HTSEXT_API __declspec(dllimport)
#endif
#else
#define HTSEXT_API
#define HTSEXT_API
#endif
#ifndef HTS_LONGLONG
@@ -320,41 +316,48 @@ Please visit our Website: http://www.httrack.com
#if HTS_LONGLONG
#ifdef LLINT_FORMAT
typedef LLINT_TYPE LLint;
typedef LLINT_TYPE TStamp;
#define LLintP LLINT_FORMAT
typedef LLINT_TYPE LLint;
typedef LLINT_TYPE TStamp;
#define LLintP LLINT_FORMAT
#else
#ifdef _WIN32
typedef __int64 LLint;
typedef __int64 TStamp;
#define LLintP "%I64d"
typedef __int64 LLint;
typedef __int64 TStamp;
#define LLintP "%I64d"
#elif (defined(__x86_64__) || defined(_LP64) || defined(__64BIT__))
typedef unsigned long int LLint;
typedef unsigned long int TStamp;
#define LLintP "%ld"
typedef unsigned long int LLint;
typedef unsigned long int TStamp;
#define LLintP "%ld"
#else
typedef long long int LLint;
typedef long long int TStamp;
#define LLintP "%lld"
typedef long long int LLint;
typedef long long int TStamp;
#define LLintP "%lld"
#endif
#endif /* HTS_LONGLONG */
#endif /* HTS_LONGLONG */
#else
typedef int LLint;
#define LLintP "%d"
typedef double TStamp;
typedef int LLint;
#define LLintP "%d"
typedef double TStamp;
#endif
#ifdef LFS_FLAG
typedef LLint INTsys;
#define INTsysP LLintP
#ifdef __linux
#define HTS_FSEEKO
#endif
#else
typedef int INTsys;
#define INTsysP "%d"
#endif
@@ -378,7 +381,7 @@ typedef int T_SOC;
#ifdef _WIN32
#else
#define __cdecl
#define __cdecl
#endif
/* mode pour mkdir ET chmod (accès aux fichiers) */
@@ -408,8 +411,6 @@ typedef int T_SOC;
/* fflush sur stdout */
#define io_flush { fflush(stdout); fflush(stdin); }
/* HTSLib */
// Cache DNS, accélère les résolution d'adresses
@@ -435,15 +436,16 @@ typedef int T_SOC;
//#define HTS_TRACE_MALLOC
#ifdef HTS_TRACE_MALLOC
typedef unsigned long int t_htsboundary;
#ifndef HTS_DEF_FWSTRUCT_mlink
#define HTS_DEF_FWSTRUCT_mlink
typedef struct mlink mlink;
#endif
struct mlink {
char* adr;
char *adr;
int len;
int id;
struct mlink* next;
struct mlink *next;
};
static const t_htsboundary htsboundary = 0xDEADBEEF;
#endif
@@ -454,7 +456,6 @@ static const t_htsboundary htsboundary = 0xDEADBEEF;
#define STRDEBUG 1
#endif
/* ------------------------------------------------------------ */
/* Debugging */
/* ------------------------------------------------------------ */
@@ -498,7 +499,6 @@ static const t_htsboundary htsboundary = 0xDEADBEEF;
// htsmain
#define DEBUG_STEPS 0
// Débuggage de contrôle
#if HTS_DEBUG_CLOSESOCK
#define _HTS_WIDE 1
@@ -507,13 +507,11 @@ static const t_htsboundary htsboundary = 0xDEADBEEF;
#define _HTS_WIDE 1
#endif
#if _HTS_WIDE
extern FILE* DEBUG_fp;
extern FILE *DEBUG_fp;
#define DEBUG_W(A) { if (DEBUG_fp==NULL) DEBUG_fp=fopen("bug.out","wb"); fprintf(DEBUG_fp,":>"A); fflush(DEBUG_fp); }
#undef _
#define _ ,
#endif
#endif

282
src/htshash.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* hash table system (fast index) */
@@ -63,68 +60,70 @@ Please visit our Website: http://www.httrack.com
// type: numero enregistrement - 0 est case insensitive (sav) 1 (adr+fil) 2 (former_adr+former_fil)
// recherche dans la table selon nom1,nom2 et le no d'enregistrement
// retour: position ou -1 si non trouvé
int hash_read(const hash_struct* hash,const char* nom1,const char* nom2,int type,int normalized) {
char BIGSTK normfil_[HTS_URLMAXSIZE*2];
char catbuff[CATBUFF_SIZE];
const char* normfil;
const char* normadr;
int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
int type, int normalized) {
char BIGSTK normfil_[HTS_URLMAXSIZE * 2];
char catbuff[CATBUFF_SIZE];
const char *normfil;
const char *normadr;
unsigned int cle;
int pos;
int pos;
// calculer la clé de recherche, non modulée
if (type)
cle = hash_cle(nom1,nom2);
cle = hash_cle(nom1, nom2);
else
cle = hash_cle(convtolower(catbuff,nom1),nom2); // case insensitive
cle = hash_cle(convtolower(catbuff, nom1), nom2); // case insensitive
// la position se calcule en modulant
pos = (int) (cle%HTS_HASH_SIZE);
pos = (int) (cle % HTS_HASH_SIZE);
// entrée trouvée?
if (hash->hash[type][pos] >= 0) { // un ou plusieurs enregistrement(s) avec une telle clé existe..
if (hash->hash[type][pos] >= 0) { // un ou plusieurs enregistrement(s) avec une telle clé existe..
// tester table de raccourcis (hash)
// pos est maintenant la position recherchée dans liens
pos = hash->hash[type][pos];
while (pos>=0) { // parcourir la chaine
while(pos >= 0) { // parcourir la chaine
switch (type) {
case 0: // sav
if (strfield2(nom1,hash->liens[pos]->sav)) { // case insensitive
case 0: // sav
if (strfield2(nom1, hash->liens[pos]->sav)) { // case insensitive
#if DEBUG_HASH==2
printf("hash: found shortcut at %d\n",pos);
printf("hash: found shortcut at %d\n", pos);
#endif
return pos;
}
break;
case 1: // adr+fil
case 1: // adr+fil
{
if (!normalized)
normfil=hash->liens[pos]->fil;
normfil = hash->liens[pos]->fil;
else
normfil=fil_normalized(hash->liens[pos]->fil,normfil_);
normfil = fil_normalized(hash->liens[pos]->fil, normfil_);
if (!normalized)
normadr = jump_identification(hash->liens[pos]->adr);
else
normadr = jump_normalized(hash->liens[pos]->adr);
if ((strfield2(nom1,normadr)!=0) && (strcmp(nom2,normfil)==0)) {
if ((strfield2(nom1, normadr) != 0) && (strcmp(nom2, normfil) == 0)) {
#if DEBUG_HASH==2
printf("hash: found shortcut at %d\n",pos);
printf("hash: found shortcut at %d\n", pos);
#endif
return pos;
}
}
break;
case 2: // former_adr+former_fil
case 2: // former_adr+former_fil
{
if (hash->liens[pos]->former_adr) {
if (!normalized)
normfil=hash->liens[pos]->former_fil;
normfil = hash->liens[pos]->former_fil;
else
normfil=fil_normalized(hash->liens[pos]->former_fil,normfil_);
normfil = fil_normalized(hash->liens[pos]->former_fil, normfil_);
if (!normalized)
normadr = jump_identification(hash->liens[pos]->former_adr);
else
normadr = jump_normalized(hash->liens[pos]->former_adr);
if ((strfield2(nom1,normadr)!=0) && (strcmp(nom2,normfil)==0)) {
if ((strfield2(nom1, normadr) != 0) && (strcmp(nom2, normfil) == 0)) {
#if DEBUG_HASH==2
printf("hash: found shortcut at %d\n",pos);
printf("hash: found shortcut at %d\n", pos);
#endif
return pos;
}
@@ -134,135 +133,140 @@ int hash_read(const hash_struct* hash,const char* nom1,const char* nom2,int type
}
// calculer prochaine position dans la chaine
{
int old=pos;
pos=hash->liens[pos]->hash_next[type]; // sinon prochain dans la chaine
if (old==pos)
pos=-1; // erreur de bouclage (ne devrait pas arriver)
int old = pos;
pos = hash->liens[pos]->hash_next[type]; // sinon prochain dans la chaine
if (old == pos)
pos = -1; // erreur de bouclage (ne devrait pas arriver)
}
}
// Ok va falloir chercher alors..
/*pos=hash->max_lien; // commencer à max_lien
switch (type) {
case 0: // sav
while(pos>=0) {
if (hash->liens[pos]->hash_sav == cle ) {
if (strcmp(nom1,hash->liens[pos]->sav)==0) {
hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table
#if DEBUG_HASH==2
printf("hash: found long search at %d\n",pos);
#endif
return pos;
}
}
pos--;
}
break;
case 1: // adr+fil
while(pos>=0) {
if (hash->liens[pos]->hash_adrfil == cle ) {
if ((strcmp(nom1,hash->liens[pos]->adr)==0) && (strcmp(nom2,hash->liens[pos]->fil)==0)) {
hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table
#if DEBUG_HASH==2
printf("hash: found long search at %d\n",pos);
#endif
return pos;
}
}
pos--;
}
break;
case 2: // former_adr+former_fil
while(pos>=0) {
if (hash->liens[pos]->hash_fadrfil == cle ) {
if (hash->liens[pos]->former_adr)
if ((strcmp(nom1,hash->liens[pos]->former_adr)==0) && (strcmp(nom2,hash->liens[pos]->former_fil)==0)) {
hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table
#if DEBUG_HASH==2
printf("hash: found long search at %d\n",pos);
#endif
return pos;
}
}
pos--;
}
}*/
switch (type) {
case 0: // sav
while(pos>=0) {
if (hash->liens[pos]->hash_sav == cle ) {
if (strcmp(nom1,hash->liens[pos]->sav)==0) {
hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table
#if DEBUG_HASH==2
printf("hash: found long search at %d\n",pos);
#endif
return pos;
}
}
pos--;
}
break;
case 1: // adr+fil
while(pos>=0) {
if (hash->liens[pos]->hash_adrfil == cle ) {
if ((strcmp(nom1,hash->liens[pos]->adr)==0) && (strcmp(nom2,hash->liens[pos]->fil)==0)) {
hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table
#if DEBUG_HASH==2
printf("hash: found long search at %d\n",pos);
#endif
return pos;
}
}
pos--;
}
break;
case 2: // former_adr+former_fil
while(pos>=0) {
if (hash->liens[pos]->hash_fadrfil == cle ) {
if (hash->liens[pos]->former_adr)
if ((strcmp(nom1,hash->liens[pos]->former_adr)==0) && (strcmp(nom2,hash->liens[pos]->former_fil)==0)) {
hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table
#if DEBUG_HASH==2
printf("hash: found long search at %d\n",pos);
#endif
return pos;
}
}
pos--;
}
} */
#if DEBUG_HASH==1
printf("hash: not found after test %s%s\n",nom1,nom2);
printf("hash: not found after test %s%s\n", nom1, nom2);
#endif
return -1; // non trouvé
return -1; // non trouvé
} else {
#if DEBUG_HASH==2
printf("hash: not found %s%s\n",nom1,nom2);
printf("hash: not found %s%s\n", nom1, nom2);
#endif
return -1; // non trouvé : clé non entrée (même une fois)
return -1; // non trouvé : clé non entrée (même une fois)
}
}
// enregistrement lien lpos dans les 3 tables hash1..3
void hash_write(hash_struct* hash,int lpos,int normalized) {
char BIGSTK normfil_[HTS_URLMAXSIZE*2];
char catbuff[CATBUFF_SIZE];
const char* normfil;
void hash_write(hash_struct * hash, int lpos, int normalized) {
char BIGSTK normfil_[HTS_URLMAXSIZE * 2];
char catbuff[CATBUFF_SIZE];
const char *normfil;
unsigned int cle;
int pos;
int* ptr;
int pos;
int *ptr;
//
if (hash->liens[lpos]) { // on sait jamais..
hash->max_lien = max(hash->max_lien,lpos);
if (hash->liens[lpos]) { // on sait jamais..
hash->max_lien = max(hash->max_lien, lpos);
#if DEBUG_HASH
hashnumber=hash->max_lien;
hashnumber = hash->max_lien;
#endif
// élément actuel sur -1 (fin de chaine)
hash->liens[lpos]->hash_next[0]=hash->liens[lpos]->hash_next[1]=hash->liens[lpos]->hash_next[2]=-1;
hash->liens[lpos]->hash_next[0] = hash->liens[lpos]->hash_next[1] =
hash->liens[lpos]->hash_next[2] = -1;
//
cle = hash_cle(convtolower(catbuff,hash->liens[lpos]->sav),""); // CASE INSENSITIVE
pos = (int) (cle%HTS_HASH_SIZE);
ptr = hash_calc_chaine(hash,0,pos); // calculer adresse chaine
*ptr = lpos; // noter dernier enregistré
cle = hash_cle(convtolower(catbuff, hash->liens[lpos]->sav), ""); // CASE INSENSITIVE
pos = (int) (cle % HTS_HASH_SIZE);
ptr = hash_calc_chaine(hash, 0, pos); // calculer adresse chaine
*ptr = lpos; // noter dernier enregistré
#if DEBUG_HASH==3
printf("[%d",pos);
printf("[%d", pos);
#endif
//
if (!normalized)
normfil=hash->liens[lpos]->fil;
normfil = hash->liens[lpos]->fil;
else
normfil=fil_normalized(hash->liens[lpos]->fil,normfil_);
normfil = fil_normalized(hash->liens[lpos]->fil, normfil_);
if (!normalized)
cle = hash_cle(jump_identification(hash->liens[lpos]->adr),normfil);
cle = hash_cle(jump_identification(hash->liens[lpos]->adr), normfil);
else
cle = hash_cle(jump_normalized(hash->liens[lpos]->adr),normfil);
pos = (int) (cle%HTS_HASH_SIZE);
ptr = hash_calc_chaine(hash,1,pos); // calculer adresse chaine
*ptr = lpos; // noter dernier enregistré
cle = hash_cle(jump_normalized(hash->liens[lpos]->adr), normfil);
pos = (int) (cle % HTS_HASH_SIZE);
ptr = hash_calc_chaine(hash, 1, pos); // calculer adresse chaine
*ptr = lpos; // noter dernier enregistré
#if DEBUG_HASH==3
printf(",%d",pos);
printf(",%d", pos);
#endif
//
if (hash->liens[lpos]->former_adr) { // former_adr existe?
if (hash->liens[lpos]->former_adr) { // former_adr existe?
if (!normalized)
normfil=hash->liens[lpos]->former_fil;
normfil = hash->liens[lpos]->former_fil;
else
normfil=fil_normalized(hash->liens[lpos]->former_fil,normfil_);
normfil = fil_normalized(hash->liens[lpos]->former_fil, normfil_);
if (!normalized)
cle = hash_cle(jump_identification(hash->liens[lpos]->former_adr),normfil);
cle =
hash_cle(jump_identification(hash->liens[lpos]->former_adr), normfil);
else
cle = hash_cle(jump_normalized(hash->liens[lpos]->former_adr),normfil);
pos = (int) (cle%HTS_HASH_SIZE);
ptr = hash_calc_chaine(hash,2,pos); // calculer adresse chaine
*ptr = lpos; // noter dernier enregistré
cle = hash_cle(jump_normalized(hash->liens[lpos]->former_adr), normfil);
pos = (int) (cle % HTS_HASH_SIZE);
ptr = hash_calc_chaine(hash, 2, pos); // calculer adresse chaine
*ptr = lpos; // noter dernier enregistré
#if DEBUG_HASH==3
printf(",%d",pos);
printf(",%d", pos);
#endif
}
#if DEBUG_HASH==3
printf("] "); fflush(stdout);
printf("] ");
fflush(stdout);
#endif
}
#if DEBUT_HASH
else {
printf("* hash_write=0!!\n");
abortLogFmt("unexpected error in hash_write (pos=%d)" _ pos);
abortLogFmt("unexpected error in hash_write (pos=%d)" _pos);
abort();
}
#endif
@@ -271,37 +275,37 @@ void hash_write(hash_struct* hash,int lpos,int normalized) {
// calcul clé
// il n'y a pas de formule de hashage universelle, celle-ci semble acceptable..
unsigned long int hash_cle(const char* nom1, const char* nom2) {
unsigned long int hash_cle(const char *nom1, const char *nom2) {
/*
unsigned int sum=0;
int i=0;
while(*nom1) {
sum += 1;
sum += (unsigned int) *(nom1);
sum *= (unsigned int) *(nom1++);
sum += (unsigned int) i;
i++;
}
while(*nom2) {
sum += 1;
sum += (unsigned int) *(nom2);
sum *= (unsigned int) *(nom2++);
sum += (unsigned int) i;
i++;
}
*/
unsigned int sum=0;
int i=0;
while(*nom1) {
sum += 1;
sum += (unsigned int) *(nom1);
sum *= (unsigned int) *(nom1++);
sum += (unsigned int) i;
i++;
}
while(*nom2) {
sum += 1;
sum += (unsigned int) *(nom2);
sum *= (unsigned int) *(nom2++);
sum += (unsigned int) i;
i++;
}
*/
return md5sum32(nom1)
+md5sum32(nom2);
+ md5sum32(nom2);
}
// calcul de la position finale dans la chaine des elements ayant la même clé
int* hash_calc_chaine(hash_struct* hash,int type,int pos) {
int *hash_calc_chaine(hash_struct * hash, int type, int pos) {
#if DEBUG_HASH
int count=0;
int count = 0;
#endif
if (hash->hash[type][pos] == -1)
return &(hash->hash[type][pos]); // premier élément dans la chaine
pos=hash->hash[type][pos];
pos = hash->hash[type][pos];
while(hash->liens[pos]->hash_next[type] != -1) {
pos = hash->liens[pos]->hash_next[type];
#if DEBUG_HASH
@@ -310,9 +314,9 @@ int* hash_calc_chaine(hash_struct* hash,int type,int pos) {
}
#if DEBUG_HASH
count++;
longest_hash[type]=max(longest_hash[type],count);
longest_hash[type] = max(longest_hash[type], count);
#endif
return &(hash->liens[pos]->hash_next[type]);
}
// FIN GESTION DES TABLES DE HACHAGE
// FIN GESTION DES TABLES DE HACHAGE

16
src/htshash.h
View File

@@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* hash table system (fast index) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSHASH_DEFH
#define HTSHASH_DEFH
#define HTSHASH_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
@@ -50,10 +45,11 @@ typedef struct hash_struct hash_struct;
#endif
// tables de hachage
int hash_read(const hash_struct* hash,const char* nom1,const char* nom2,int type,int normalized);
void hash_write(hash_struct* hash,int lpos,int normalized);
int* hash_calc_chaine(hash_struct* hash,int type,int pos);
unsigned long int hash_cle(const char* nom1,const char* nom2);
int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
int type, int normalized);
void hash_write(hash_struct * hash, int lpos, int normalized);
int *hash_calc_chaine(hash_struct * hash, int type, int pos);
unsigned long int hash_cle(const char *nom1, const char *nom2);
#endif
#endif

585
src/htshelp.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* command-line help system */
@@ -56,50 +53,53 @@ Please visit our Website: http://www.httrack.com
/* END specific definitions */
#define waitkey if (more) { char s[4]; printf("\nMORE.. q to quit\n"); linput(stdin,s,4); if (strcmp(s,"q")==0) quit=1; else printf("Page %d\n\n",++m); }
void infomsg(char* msg) {
int l=0;
int m=0;
int more=0;
int quit=0;
int done=0;
void infomsg(char *msg) {
int l = 0;
int m = 0;
int more = 0;
int quit = 0;
int done = 0;
//
if (msg==NULL)
quit=0;
if (msg == NULL)
quit = 0;
if (msg) {
if (!quit) {
if (strlen(msg)==1) {
if (msg[0]=='1') {
more=1;
if (strlen(msg) == 1) {
if (msg[0] == '1') {
more = 1;
return;
}
}
/* afficher alias? */
if (((int)strlen(msg)) > 4) {
if (msg[0]==' ') {
if (msg[2]!=' ') {
if ((msg[3]==' ') || (msg[4]==' ')) {
char cmd[32]="-";
int p=0;
while(cmd[p]==' ') p++;
sscanf(msg+p,"%s",cmd+strlen(cmd));
if (((int) strlen(msg)) > 4) {
if (msg[0] == ' ') {
if (msg[2] != ' ') {
if ((msg[3] == ' ') || (msg[4] == ' ')) {
char cmd[32] = "-";
int p = 0;
while(cmd[p] == ' ')
p++;
sscanf(msg + p, "%s", cmd + strlen(cmd));
/* clears cN -> c */
if ((p = (int) strlen(cmd))>2)
if (cmd[p-1]=='N')
cmd[p-1]='\0';
if ((p = (int) strlen(cmd)) > 2)
if (cmd[p - 1] == 'N')
cmd[p - 1] = '\0';
/* finds alias (if any) */
p=optreal_find(cmd);
if (p>=0) {
p = optreal_find(cmd);
if (p >= 0) {
/* fings type of parameter: number,param,param concatenated,single cmd */
if (strcmp(opttype_value(p),"param") == 0)
printf("%s (--%s[=N])\n",msg,optalias_value(p));
else if (strcmp(opttype_value(p),"param1") == 0)
printf("%s (--%s <param>)\n",msg,optalias_value(p));
else if (strcmp(opttype_value(p),"param0") == 0)
printf("%s (--%s<param>)\n",msg,optalias_value(p));
if (strcmp(opttype_value(p), "param") == 0)
printf("%s (--%s[=N])\n", msg, optalias_value(p));
else if (strcmp(opttype_value(p), "param1") == 0)
printf("%s (--%s <param>)\n", msg, optalias_value(p));
else if (strcmp(opttype_value(p), "param0") == 0)
printf("%s (--%s<param>)\n", msg, optalias_value(p));
else
printf("%s (--%s)\n",msg,optalias_value(p));
done=1;
printf("%s (--%s)\n", msg, optalias_value(p));
done = 1;
}
}
}
@@ -108,182 +108,214 @@ void infomsg(char* msg) {
/* sinon */
if (!done)
printf("%s\n",msg);
printf("%s\n", msg);
l++;
if (l>20) {
l=0;
if (l > 20) {
l = 0;
waitkey;
}
}
}
}
void help_wizard(httrackp* opt) {
char* urls = (char*) malloct(HTS_URLMAXSIZE*2);
char* mainpath = (char*) malloct(256);
char* projname = (char*) malloct(256);
char* stropt = (char*) malloct(2048); // options
char* stropt2 = (char*) malloct(2048); // options longues
char* strwild = (char*) malloct(2048); // wildcards
char* cmd = (char*) malloct(4096);
char* str = (char*) malloct(256);
char** argv = (char**) malloct(256 * sizeof(char*));
void help_wizard(httrackp * opt) {
char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
char *mainpath = (char *) malloct(256);
char *projname = (char *) malloct(256);
char *stropt = (char *) malloct(2048); // options
char *stropt2 = (char *) malloct(2048); // options longues
char *strwild = (char *) malloct(2048); // wildcards
char *cmd = (char *) malloct(4096);
char *str = (char *) malloct(256);
char **argv = (char **) malloct(256 * sizeof(char *));
//
char* a;
char *a;
//
if (urls == NULL || mainpath == NULL || projname == NULL || stropt == NULL
|| stropt2 == NULL || strwild == NULL || cmd == NULL || str == NULL || argv == NULL) {
if (urls == NULL || mainpath == NULL || projname == NULL || stropt == NULL
|| stropt2 == NULL || strwild == NULL || cmd == NULL || str == NULL
|| argv == NULL) {
fprintf(stderr, "* memory exhausted in %s, line %d\n", __FILE__, __LINE__);
return;
}
urls[0] = mainpath[0] = projname[0] = stropt[0] = stropt2[0] = strwild[0] = cmd[0] = str[0] = '\0';
urls[0] = mainpath[0] = projname[0] = stropt[0] = stropt2[0] = strwild[0] =
cmd[0] = str[0] = '\0';
//
strcpybuff(stropt,"-");
mainpath[0]=projname[0]=stropt2[0]=strwild[0]='\0';
strcpybuff(stropt, "-");
mainpath[0] = projname[0] = stropt2[0] = strwild[0] = '\0';
//
printf("\n");
printf("Welcome to HTTrack Website Copier (Offline Browser) "HTTRACK_VERSION"%s\n", hts_get_version_info(opt));
printf("Welcome to HTTrack Website Copier (Offline Browser) " HTTRACK_VERSION
"%s\n", hts_get_version_info(opt));
printf("Copyright (C) Xavier Roche and other contributors\n");
#ifdef _WIN32
printf("Note: You are running the commandline version,\n");
printf("run 'WinHTTrack.exe' to get the GUI version.\n");
#endif
#ifdef HTTRACK_AFF_WARNING
printf("NOTE: "HTTRACK_AFF_WARNING"\n");
printf("NOTE: " HTTRACK_AFF_WARNING "\n");
#endif
#ifdef HTS_PLATFORM_NAME
#if USE_BEGINTHREAD
printf("[compiled: "HTS_PLATFORM_NAME" - MT]\n");
printf("[compiled: " HTS_PLATFORM_NAME " - MT]\n");
#else
printf("[compiled: "HTS_PLATFORM_NAME"]\n");
printf("[compiled: " HTS_PLATFORM_NAME "]\n");
#endif
#endif
printf("To see the option list, enter a blank line or try httrack --help\n");
//
// Project name
while(strnotempty(projname)==0) {
while(strnotempty(projname) == 0) {
printf("\n");
printf("Enter project name :");
fflush(stdout);
linput(stdin,projname,250);
if (strnotempty(projname)==0)
help("httrack",1);
linput(stdin, projname, 250);
if (strnotempty(projname) == 0)
help("httrack", 1);
}
//
// Path
if (strnotempty(hts_gethome()))
printf("\nBase path (return=%s/websites/) :",hts_gethome());
printf("\nBase path (return=%s/websites/) :", hts_gethome());
else
printf("\nBase path (return=current directory) :");
linput(stdin,str,250);
linput(stdin, str, 250);
if (!strnotempty(str)) {
strcatbuff(str,hts_gethome());
strcatbuff(str,"/websites/");
strcatbuff(str, hts_gethome());
strcatbuff(str, "/websites/");
}
if (strnotempty(str))
if ((str[strlen(str)-1]!='/') && (str[strlen(str)-1]!='\\'))
strcatbuff(str,"/");
strcatbuff(stropt2,"-O \""); strcatbuff(stropt2,str); strcatbuff(stropt2,projname); strcatbuff(stropt2,"\" ");
if ((str[strlen(str) - 1] != '/') && (str[strlen(str) - 1] != '\\'))
strcatbuff(str, "/");
strcatbuff(stropt2, "-O \"");
strcatbuff(stropt2, str);
strcatbuff(stropt2, projname);
strcatbuff(stropt2, "\" ");
// Créer si ce n'est fait un index.html 1er niveau
make_empty_index(str);
//
printf("\n");
printf("Enter URLs (separated by commas or blank spaces) :");
fflush(stdout);
linput(stdin,urls,250);
linput(stdin, urls, 250);
if (strnotempty(urls)) {
while( (a=strchr(urls,',')) ) *a=' ';
while( (a=strchr(urls,'\t')) ) *a=' ';
while((a = strchr(urls, ',')))
*a = ' ';
while((a = strchr(urls, '\t')))
*a = ' ';
// Action
printf("\nAction:\n");
switch(help_query("Mirror Web Site(s)|Mirror Web Site(s) with Wizard|Just Get Files Indicated|Mirror ALL links in URLs (Multiple Mirror)|Test Links In URLs (Bookmark Test)|Update/Continue a Mirror",1)) {
case 1: break;
case 2: strcatbuff(stropt,"W"); break;
case 3: strcatbuff(stropt2,"--get "); break;
case 4: strcatbuff(stropt2,"--mirrorlinks "); break;
case 5: strcatbuff(stropt2,"--testlinks "); break;
case 6: strcatbuff(stropt2,"--update "); break;
case 0: return; break;
switch (help_query
("Mirror Web Site(s)|Mirror Web Site(s) with Wizard|Just Get Files Indicated|Mirror ALL links in URLs (Multiple Mirror)|Test Links In URLs (Bookmark Test)|Update/Continue a Mirror",
1)) {
case 1:
break;
case 2:
strcatbuff(stropt, "W");
break;
case 3:
strcatbuff(stropt2, "--get ");
break;
case 4:
strcatbuff(stropt2, "--mirrorlinks ");
break;
case 5:
strcatbuff(stropt2, "--testlinks ");
break;
case 6:
strcatbuff(stropt2, "--update ");
break;
case 0:
return;
break;
}
// Proxy
printf("\nProxy (return=none) :");
linput(stdin,str,250);
linput(stdin, str, 250);
if (strnotempty(str)) {
while( (a=strchr(str,' ')) ) *a=':'; // port
if (!strchr(jump_identification(str),':')) {
while((a = strchr(str, ' ')))
*a = ':'; // port
if (!strchr(jump_identification(str), ':')) {
char str2[256];
printf("\nProxy port (return=8080) :");
linput(stdin,str2,250);
strcatbuff(str,":");
if (strnotempty(str2)==0)
strcatbuff(str,"8080");
linput(stdin, str2, 250);
strcatbuff(str, ":");
if (strnotempty(str2) == 0)
strcatbuff(str, "8080");
else
strcatbuff(str,str2);
strcatbuff(str, str2);
}
strcatbuff(stropt2,"-P "); strcatbuff(stropt2,str); strcatbuff(stropt2," ");
strcatbuff(stropt2, "-P ");
strcatbuff(stropt2, str);
strcatbuff(stropt2, " ");
}
// Display
strcatbuff(stropt2," -%v ");
strcatbuff(stropt2, " -%v ");
// Wildcards
printf("\nYou can define wildcards, like: -*.gif +www.*.com/*.zip -*img_*.zip\n");
printf
("\nYou can define wildcards, like: -*.gif +www.*.com/*.zip -*img_*.zip\n");
printf("Wildcards (return=none) :");
linput(stdin,strwild,250);
linput(stdin, strwild, 250);
// Options
do {
printf("\nYou can define additional options, such as recurse level (-r<number>), separed by blank spaces\n");
printf
("\nYou can define additional options, such as recurse level (-r<number>), separed by blank spaces\n");
printf("To see the option list, type help\n");
printf("Additional options (return=none) :");
linput(stdin,str,250);
if (strfield2(str,"help")) {
help("httrack",2);
linput(stdin, str, 250);
if (strfield2(str, "help")) {
help("httrack", 2);
} else if (strnotempty(str)) {
strcatbuff(stropt2,str);
strcatbuff(stropt2," ");
strcatbuff(stropt2, str);
strcatbuff(stropt2, " ");
}
} while(strfield2(str,"help"));
} while(strfield2(str, "help"));
{
int argc=1;
int g=0;
int i=0;
int argc = 1;
int g = 0;
int i = 0;
//
printf("\n");
if (strlen(stropt)==1)
stropt[0]='\0'; // aucune
sprintf(cmd,"%s %s %s %s",urls,stropt,stropt2,strwild);
printf("---> Wizard command line: httrack %s\n\n",cmd);
if (strlen(stropt) == 1)
stropt[0] = '\0'; // aucune
sprintf(cmd, "%s %s %s %s", urls, stropt, stropt2, strwild);
printf("---> Wizard command line: httrack %s\n\n", cmd);
printf("Ready to launch the mirror? (Y/n) :");
fflush(stdout);
linput(stdin,str,250);
linput(stdin, str, 250);
if (strnotempty(str)) {
if (!((str[0]=='y') || (str[0]=='Y')))
return ;
if (!((str[0] == 'y') || (str[0] == 'Y')))
return;
}
printf("\n");
// couper en morceaux
argv[0]="winhttrack";
argv[1]=cmd;
argc++;
while(cmd[i]) {
if(cmd[i]=='\"') g=!g;
if(cmd[i]==' '){
if(!g){
cmd[i]='\0';
argv[argc++]=cmd+i+1;
argv[0] = "winhttrack";
argv[1] = cmd;
argc++;
while(cmd[i]) {
if (cmd[i] == '\"')
g = !g;
if (cmd[i] == ' ') {
if (!g) {
cmd[i] = '\0';
argv[argc++] = cmd + i + 1;
}
}
}
i++;
}
hts_main(argc,argv);
hts_main(argc, argv);
}
//} else {
// help("httrack",1);
//} else {
// help("httrack",1);
}
/* Free buffers */
@@ -296,33 +328,36 @@ void help_wizard(httrackp* opt) {
freet(cmd);
freet(str);
}
int help_query(char* list,int def) {
int help_query(char *list, int def) {
char s[256];
char* a;
char *a;
int opt;
int n=1;
a=list;
int n = 1;
a = list;
while(strnotempty(a)) {
char* b = strchr(a,'|');
char *b = strchr(a, '|');
if (b) {
char str[256];
str[0]='\0';
str[0] = '\0';
//
strncatbuff(str,a,(int) (b - a));
if (n==def)
printf("(enter)\t%d\t%s\n",n++,str);
strncatbuff(str, a, (int) (b - a));
if (n == def)
printf("(enter)\t%d\t%s\n", n++, str);
else
printf("\t%d\t%s\n",n++,str);
a=b+1;
printf("\t%d\t%s\n", n++, str);
a = b + 1;
} else
a=list+strlen(list);
a = list + strlen(list);
}
printf("\t0\tQuit");
do {
printf("\n: ");
fflush(stdout);
linput(stdin,s,250);
} while ((strnotempty(s)!=0) && (sscanf(s,"%d",&opt)!=1));
linput(stdin, s, 250);
} while((strnotempty(s) != 0) && (sscanf(s, "%d", &opt) != 1));
if (strnotempty(s))
return opt;
else
@@ -330,40 +365,47 @@ int help_query(char* list,int def) {
}
// Capture d'URL
void help_catchurl(const char* dest_path) {
char BIGSTK adr_prox[HTS_URLMAXSIZE*2];
void help_catchurl(const char *dest_path) {
char BIGSTK adr_prox[HTS_URLMAXSIZE * 2];
int port_prox;
T_SOC soc=catch_url_init_std(&port_prox,adr_prox);
if (soc!=INVALID_SOCKET) {
char BIGSTK url[HTS_URLMAXSIZE*2];
T_SOC soc = catch_url_init_std(&port_prox, adr_prox);
if (soc != INVALID_SOCKET) {
char BIGSTK url[HTS_URLMAXSIZE * 2];
char method[32];
char BIGSTK data[32768];
url[0]=method[0]=data[0]='\0';
url[0] = method[0] = data[0] = '\0';
//
printf("Okay, temporary proxy installed.\nSet your browser's preferences to:\n\n");
printf("\tProxy's address: \t%s\n\tProxy's port: \t%d\n",adr_prox,port_prox);
printf
("Okay, temporary proxy installed.\nSet your browser's preferences to:\n\n");
printf("\tProxy's address: \t%s\n\tProxy's port: \t%d\n", adr_prox,
port_prox);
//
if (catch_url(soc,url,method,data)) {
char BIGSTK dest[HTS_URLMAXSIZE*2];
int i=0;
if (catch_url(soc, url, method, data)) {
char BIGSTK dest[HTS_URLMAXSIZE * 2];
int i = 0;
do {
sprintf(dest,"%s%s%d",dest_path,"hts-post",i);
sprintf(dest, "%s%s%d", dest_path, "hts-post", i);
i++;
} while(fexist(dest));
{
FILE* fp=fopen(dest,"wb");
FILE *fp = fopen(dest, "wb");
if (fp) {
fwrite(data,strlen(data),1,fp);
fwrite(data, strlen(data), 1, fp);
fclose(fp);
}
}
// former URL!
{
char BIGSTK finalurl[HTS_URLMAXSIZE*2];
char BIGSTK finalurl[HTS_URLMAXSIZE * 2];
escape_check_url(dest);
sprintf(finalurl,"%s"POSTTOK"file:%s",url,dest);
printf("\nThe URL is: \"%s\"\n",finalurl);
printf("You can capture it through: httrack \"%s\"\n",finalurl);
sprintf(finalurl, "%s" POSTTOK "file:%s", url, dest);
printf("\nThe URL is: \"%s\"\n", finalurl);
printf("You can capture it through: httrack \"%s\"\n", finalurl);
}
} else
printf("Unable to analyse the URL\n");
@@ -377,13 +419,16 @@ void help_catchurl(const char* dest_path) {
}
// Créer un index.html vide
void make_empty_index(char* str) {
void make_empty_index(char *str) {
#if 0
if (!fexist(fconcat(str,"index.html"))) {
FILE* fp=fopen(fconcat(str,"index.html"),"wb");
if (!fexist(fconcat(str, "index.html"))) {
FILE *fp = fopen(fconcat(str, "index.html"), "wb");
if (fp) {
fprintf(fp,"<!-- "HTS_TOPINDEX" -->"CRLF);
fprintf(fp,"<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"CRLF);
fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
fprintf(fp,
"<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
CRLF);
fclose(fp);
}
}
@@ -392,24 +437,30 @@ void make_empty_index(char* str) {
// mini-aide (h: help)
// y
void help(char* app,int more) {
void help(char *app, int more) {
char info[2048];
infomsg("");
if (more)
infomsg("1");
if (more != 2) {
sprintf(info, "HTTrack version "HTTRACK_VERSION"%s (compiled "__DATE__")", hts_is_available());
sprintf(info,
"HTTrack version " HTTRACK_VERSION "%s (compiled " __DATE__ ")",
hts_is_available());
infomsg(info);
#ifdef HTTRACK_AFF_WARNING
infomsg("NOTE: "HTTRACK_AFF_WARNING);
infomsg("NOTE: " HTTRACK_AFF_WARNING);
#endif
sprintf(info,"\tusage: %s <URLs> [-option] [+<URL_FILTER>] [-<URL_FILTER>] [+<mime:MIME_FILTER>] [-<mime:MIME_FILTER>]",app);
sprintf(info,
"\tusage: %s <URLs> [-option] [+<URL_FILTER>] [-<URL_FILTER>] [+<mime:MIME_FILTER>] [-<mime:MIME_FILTER>]",
app);
infomsg(info);
infomsg("\twith options listed below: (* is the default value)");
infomsg("");
}
infomsg("General options:");
infomsg(" O path for mirror/logfiles+cache (-O path_mirror[,path_cache_and_logfiles])");
infomsg
(" O path for mirror/logfiles+cache (-O path_mirror[,path_cache_and_logfiles])");
#ifndef _WIN32
infomsg(" %O chroot path to, must be r00t (-%O root_path)");
#endif
@@ -419,7 +470,8 @@ void help(char* app,int more) {
infomsg(" W mirror web sites, semi-automatic (asks questions)");
infomsg(" g just get files (saved in the current directory)");
infomsg(" i continue an interrupted mirror using the cache");
infomsg(" Y mirror ALL links located in the first level pages (mirror links)");
infomsg
(" Y mirror ALL links located in the first level pages (mirror links)");
infomsg("");
infomsg("Proxy options:");
infomsg(" P proxy use (-P proxy:port or -P user:pass@proxy:port)");
@@ -435,69 +487,103 @@ void help(char* app,int more) {
infomsg(" EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)");
infomsg(" AN maximum transfer rate in bytes/seconds (1000=1KB/s max)");
infomsg(" %cN maximum number of connections/seconds (*%c10)");
infomsg(" GN pause transfer if N bytes reached, and wait until lock file is deleted");
infomsg
(" GN pause transfer if N bytes reached, and wait until lock file is deleted");
#if HTS_USEMMS
infomsg(" %mN maximum mms stream download time in seconds (60=1 minute, 3600=1 hour)");
infomsg
(" %mN maximum mms stream download time in seconds (60=1 minute, 3600=1 hour)");
#endif
infomsg("");
infomsg("Flow control:");
infomsg(" cN number of multiple connections (*c8)");
infomsg(" TN timeout, number of seconds after a non-responding link is shutdown");
infomsg(" RN number of retries, in case of timeout or non-fatal errors (*R1)");
infomsg(" JN traffic jam control, minimum transfert rate (bytes/seconds) tolerated for a link");
infomsg(" HN host is abandonned if: 0=never, 1=timeout, 2=slow, 3=timeout or slow");
infomsg
(" TN timeout, number of seconds after a non-responding link is shutdown");
infomsg
(" RN number of retries, in case of timeout or non-fatal errors (*R1)");
infomsg
(" JN traffic jam control, minimum transfert rate (bytes/seconds) tolerated for a link");
infomsg
(" HN host is abandonned if: 0=never, 1=timeout, 2=slow, 3=timeout or slow");
infomsg("");
infomsg("Links options:");
infomsg(" %P *extended parsing, attempt to parse all links, even in unknown tags or Javascript (%P0 don't use)");
infomsg(" n get non-html files 'near' an html file (ex: an image located outside)");
infomsg
(" %P *extended parsing, attempt to parse all links, even in unknown tags or Javascript (%P0 don't use)");
infomsg
(" n get non-html files 'near' an html file (ex: an image located outside)");
infomsg(" t test all URLs (even forbidden ones)");
infomsg(" %L <file> add all URL located in this text file (one URL per line)");
infomsg(" %S <file> add all scan rules located in this text file (one scan rule per line)");
infomsg
(" %L <file> add all URL located in this text file (one URL per line)");
infomsg
(" %S <file> add all scan rules located in this text file (one scan rule per line)");
infomsg("");
infomsg("Build options:");
infomsg(" NN structure type (0 *original structure, 1+: see below)");
infomsg(" or user defined structure (-N \"%h%p/%n%q.%t\")");
infomsg(" %N delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use)");
infomsg(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
infomsg
(" %N delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use)");
infomsg
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
infomsg(" LN long names (L1 *long names / L0 8-3 conversion / L2 ISO9660 compatible)");
infomsg(" KN keep original links (e.g. http://www.adr/link) (K0 *relative link, K absolute links, K4 original links, K3 absolute URI links, K5 transparent proxy link)");
infomsg
(" LN long names (L1 *long names / L0 8-3 conversion / L2 ISO9660 compatible)");
infomsg
(" KN keep original links (e.g. http://www.adr/link) (K0 *relative link, K absolute links, K4 original links, K3 absolute URI links, K5 transparent proxy link)");
infomsg(" x replace external html links by error pages");
infomsg(" %x do not include any password for external password protected websites (%x0 include)");
infomsg(" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
infomsg(" o *generate output html file in case of error (404..) (o0 don't generate)");
infomsg
(" %x do not include any password for external password protected websites (%x0 include)");
infomsg
(" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
infomsg
(" o *generate output html file in case of error (404..) (o0 don't generate)");
infomsg(" X *purge old files after update (X0 keep delete)");
infomsg(" %p preserve html files 'as is' (identical to '-K4 -%F \"\"')");
infomsg(" %T links conversion to UTF-8");
infomsg("");
infomsg("Spider options:");
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
infomsg(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
infomsg(" j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
infomsg(" sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
infomsg(" %h force HTTP/1.0 requests (reduce update features, only for old servers or proxies)");
infomsg(" %k use keep-alive if possible, greately reducing latency for small files and test requests (%k0 don't use)");
infomsg(" %B tolerant requests (accept bogus responses on some servers, but not standard!)");
infomsg(" %s update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)");
infomsg(" %u url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)");
infomsg(" %A assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)");
infomsg(" shortcut: '--assume standard' is equivalent to -%A "HTS_ASSUME_STANDARD);
infomsg(" can also be used to force a specific file type: --assume foo.cgi=text/html");
infomsg(" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
infomsg(" %w disable a specific external mime module (-%w htsswf -%w htsjava)");
infomsg
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
infomsg
(" j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
infomsg
(" sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
infomsg
(" %h force HTTP/1.0 requests (reduce update features, only for old servers or proxies)");
infomsg
(" %k use keep-alive if possible, greately reducing latency for small files and test requests (%k0 don't use)");
infomsg
(" %B tolerant requests (accept bogus responses on some servers, but not standard!)");
infomsg
(" %s update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)");
infomsg
(" %u url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)");
infomsg
(" %A assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)");
infomsg(" shortcut: '--assume standard' is equivalent to -%A "
HTS_ASSUME_STANDARD);
infomsg
(" can also be used to force a specific file type: --assume foo.cgi=text/html");
infomsg
(" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
infomsg
(" %w disable a specific external mime module (-%w htsswf -%w htsjava)");
infomsg("");
infomsg("Browser ID:");
infomsg(" F user-agent field sent in HTTP headers (-F \"user-agent name\")");
infomsg
(" F user-agent field sent in HTTP headers (-F \"user-agent name\")");
infomsg(" %R default referer field sent in HTTP headers");
infomsg(" %E from email address sent in HTTP headers");
infomsg(" %F footer string in Html code (-%F \"Mirrored [from host %s [file %s [at %s]]]\"");
infomsg
(" %F footer string in Html code (-%F \"Mirrored [from host %s [file %s [at %s]]]\"");
infomsg(" %l preffered language (-%l \"fr, en, jp, *\"");
infomsg("");
infomsg("Log, index, cache");
infomsg(" C create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)");
infomsg
(" C create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)");
infomsg(" k store all files in cache (not useful if files on disk)");
infomsg(" %n do not re-download locally erased files");
infomsg(" %v display on screen filenames downloaded (in realtime) - * %v1 short version - %v2 full animation");
infomsg
(" %v display on screen filenames downloaded (in realtime) - * %v1 short version - %v2 full animation");
infomsg(" Q no log - quiet mode");
infomsg(" q no questions - quiet mode");
infomsg(" z log - extra infos");
@@ -517,7 +603,7 @@ void help(char* app,int more) {
infomsg(" *p3 save all files");
infomsg(" p7 get html files before, then treat other files");
infomsg(" S stay on the same directory");
infomsg(" D *can only go down into subdirs");
infomsg(" D *can only go down into subdirs");
infomsg(" U can only go to upper directories");
infomsg(" B can both go up&down into the directory structure");
infomsg(" a *stay on the same address");
@@ -548,15 +634,19 @@ void help(char* app,int more) {
infomsg(" #Z generate transfer rate statictics every minutes");
infomsg(" #! execute a shell command (-#! \"echo hello\")");
infomsg("");
infomsg("Dangerous options: (do NOT use unless you exactly know what you are doing)");
infomsg(" %! bypass built-in security limits aimed to avoid bandwith abuses (bandwidth, simultaneous connections)");
infomsg
("Dangerous options: (do NOT use unless you exactly know what you are doing)");
infomsg
(" %! bypass built-in security limits aimed to avoid bandwith abuses (bandwidth, simultaneous connections)");
infomsg(" IMPORTANT NOTE: DANGEROUS OPTION, ONLY SUITABLE FOR EXPERTS");
infomsg(" USE IT WITH EXTREME CARE");
infomsg("");
infomsg("Command-line specific options:");
infomsg(" V execute system command after each files ($0 is the filename: -V \"rm \\$0\")");
infomsg
(" V execute system command after each files ($0 is the filename: -V \"rm \\$0\")");
infomsg(" %U run the engine with another id when called as root (-%U smith)");
infomsg(" %W use an external library function as a wrapper (-%W myfoo.so[,myparameters])");
infomsg
(" %W use an external library function as a wrapper (-%W myfoo.so[,myparameters])");
/* infomsg(" %O do a chroot before setuid"); */
infomsg("");
infomsg("Details: Option N");
@@ -564,19 +654,27 @@ void help(char* app,int more) {
infomsg(" N1 HTML in web/, images/other files in web/images/");
infomsg(" N2 HTML in web/HTML, images/other in web/images");
infomsg(" N3 HTML in web/, images/other in web/");
infomsg(" N4 HTML in web/, images/other in web/xxx, where xxx is the file extension (all gif will be placed onto web/gif, for example)");
infomsg
(" N4 HTML in web/, images/other in web/xxx, where xxx is the file extension (all gif will be placed onto web/gif, for example)");
infomsg(" N5 Images/other in web/xxx and HTML in web/HTML");
infomsg(" N99 All files in web/, with random names (gadget !)");
infomsg(" N100 Site-structure, without www.domain.xxx/");
infomsg(" N101 Identical to N1 exept that \"web\" is replaced by the site's name");
infomsg(" N102 Identical to N2 exept that \"web\" is replaced by the site's name");
infomsg(" N103 Identical to N3 exept that \"web\" is replaced by the site's name");
infomsg(" N104 Identical to N4 exept that \"web\" is replaced by the site's name");
infomsg(" N105 Identical to N5 exept that \"web\" is replaced by the site's name");
infomsg(" N199 Identical to N99 exept that \"web\" is replaced by the site's name");
infomsg
(" N101 Identical to N1 exept that \"web\" is replaced by the site's name");
infomsg
(" N102 Identical to N2 exept that \"web\" is replaced by the site's name");
infomsg
(" N103 Identical to N3 exept that \"web\" is replaced by the site's name");
infomsg
(" N104 Identical to N4 exept that \"web\" is replaced by the site's name");
infomsg
(" N105 Identical to N5 exept that \"web\" is replaced by the site's name");
infomsg
(" N199 Identical to N99 exept that \"web\" is replaced by the site's name");
infomsg(" N1001 Identical to N1 exept that there is no \"web\" directory");
infomsg(" N1002 Identical to N2 exept that there is no \"web\" directory");
infomsg(" N1003 Identical to N3 exept that there is no \"web\" directory (option set for g option)");
infomsg
(" N1003 Identical to N3 exept that there is no \"web\" directory (option set for g option)");
infomsg(" N1004 Identical to N4 exept that there is no \"web\" directory");
infomsg(" N1005 Identical to N5 exept that there is no \"web\" directory");
infomsg(" N1099 Identical to N99 exept that there is no \"web\" directory");
@@ -593,36 +691,46 @@ void help(char* app,int more) {
infomsg(" '%q' small query string MD5 (16 bits, 4 ascii bytes)");
infomsg(" '%s?' Short name version (ex: %sN)");
infomsg(" '%[param]' param variable in query string");
infomsg(" '%[param:before:after:empty:notfound]' advanced variable extraction");
infomsg
(" '%[param:before:after:empty:notfound]' advanced variable extraction");
infomsg("Details: User-defined option N and advanced variable extraction");
infomsg(" %[param:before:after:empty:notfound]");
infomsg(" %[param:before:after:empty:notfound]");
infomsg(" param : parameter name");
infomsg(" before : string to prepend if the parameter was found");
infomsg(" after : string to append if the parameter was found");
infomsg(" notfound : string replacement if the parameter could not be found");
infomsg
(" notfound : string replacement if the parameter could not be found");
infomsg(" empty : string replacement if the parameter was empty");
infomsg(" all fields, except the first one (the parameter name), can be empty");
infomsg
(" all fields, except the first one (the parameter name), can be empty");
infomsg("");
infomsg("Details: Option K");
infomsg(" K0 foo.cgi?q=45 -> foo4B54.html?q=45 (relative URI, default)");
infomsg(" K -> http://www.foobar.com/folder/foo.cgi?q=45 (absolute URL)");
infomsg
(" K -> http://www.foobar.com/folder/foo.cgi?q=45 (absolute URL)");
infomsg(" K3 -> /folder/foo.cgi?q=45 (absolute URI)");
infomsg(" K4 -> foo.cgi?q=45 (original URL)");
infomsg(" K5 -> http://www.foobar.com/folder/foo4B54.html?q=45 (transparent proxy URL)");
infomsg
(" K5 -> http://www.foobar.com/folder/foo4B54.html?q=45 (transparent proxy URL)");
infomsg("");
infomsg("Shortcuts:");
infomsg("--mirror <URLs> *make a mirror of site(s) (default)");
infomsg("--get <URLs> get the files indicated, do not seek other URLs (-qg)");
infomsg
("--get <URLs> get the files indicated, do not seek other URLs (-qg)");
infomsg("--list <text file> add all URL located in this text file (-%L)");
infomsg("--mirrorlinks <URLs> mirror all links in 1st level pages (-Y)");
infomsg("--testlinks <URLs> test links in pages (-r1p0C0I0t)");
infomsg("--spider <URLs> spider site(s), to test links: reports Errors & Warnings (-p0C0I0t)");
infomsg
("--spider <URLs> spider site(s), to test links: reports Errors & Warnings (-p0C0I0t)");
infomsg("--testsite <URLs> identical to --spider");
infomsg("--skeleton <URLs> make a mirror, but gets only html files (-p1)");
infomsg
("--skeleton <URLs> make a mirror, but gets only html files (-p1)");
infomsg("--update update a mirror, without confirmation (-iC2)");
infomsg("--continue continue a mirror, without confirmation (-iC1)");
infomsg
("--continue continue a mirror, without confirmation (-iC1)");
infomsg("");
infomsg("--catchurl create a temporary proxy to capture an URL or a form post URL");
infomsg
("--catchurl create a temporary proxy to capture an URL or a form post URL");
infomsg("--clean erase cache & log files");
infomsg("");
infomsg("--http10 force http/1.0 requests (-%h)");
@@ -633,13 +741,17 @@ void help(char* app,int more) {
infomsg("example: httrack www.someweb.com/bob/");
infomsg("means: mirror site www.someweb.com/bob/ and only this site");
infomsg("");
infomsg("example: httrack www.someweb.com/bob/ www.anothertest.com/mike/ +*.com/*.jpg -mime:application/*");
infomsg("means: mirror the two sites together (with shared links) and accept any .jpg files on .com sites");
infomsg
("example: httrack www.someweb.com/bob/ www.anothertest.com/mike/ +*.com/*.jpg -mime:application/*");
infomsg
("means: mirror the two sites together (with shared links) and accept any .jpg files on .com sites");
infomsg("");
infomsg("example: httrack www.someweb.com/bob/bobby.html +* -r6");
infomsg("means get all files starting from bobby.html, with 6 link-depth, and possibility of going everywhere on the web");
infomsg
("means get all files starting from bobby.html, with 6 link-depth, and possibility of going everywhere on the web");
infomsg("");
infomsg("example: httrack www.someweb.com/bob/bobby.html --spider -P proxy.myhost.com:8080");
infomsg
("example: httrack www.someweb.com/bob/bobby.html --spider -P proxy.myhost.com:8080");
infomsg("runs the spider on www.someweb.com/bob/bobby.html using a proxy");
infomsg("");
infomsg("example: httrack --update");
@@ -651,16 +763,15 @@ void help(char* app,int more) {
infomsg("example: httrack --continue");
infomsg("continues a mirror in the current folder");
infomsg("");
sprintf(info, "HTTrack version "HTTRACK_VERSION"%s (compiled "__DATE__")", hts_is_available());
sprintf(info, "HTTrack version " HTTRACK_VERSION "%s (compiled " __DATE__ ")",
hts_is_available());
infomsg(info);
infomsg("Copyright (C) Xavier Roche and other contributors");
#ifdef HTS_PLATFORM_NAME
infomsg("[compiled: "HTS_PLATFORM_NAME"]");
infomsg("[compiled: " HTS_PLATFORM_NAME "]");
#endif
infomsg(NULL);
// infomsg(" R *relative links (e.g ../link)\n");
// infomsg(" A absolute links (e.g /www.adr/link)\n");
}

19
src/htshelp.h
View File

@@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* command-line help system */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSHELP_DEFH
#define HTSHELP_DEFH
#define HTSHELP_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
@@ -49,12 +44,12 @@ Please visit our Website: http://www.httrack.com
typedef struct httrackp httrackp;
#endif
void infomsg(char* msg);
void help(char* app,int more);
void make_empty_index(char* str);
void help_wizard(httrackp* opt);
int help_query(char* list,int def);
void help_catchurl(const char* dest_path);
void infomsg(char *msg);
void help(char *app, int more);
void make_empty_index(char *str);
void help_wizard(httrackp * opt);
int help_query(char *list, int def);
void help_catchurl(const char *dest_path);
#endif

398
src/htsindex.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsindex.c */
/* keyword indexing system (search index) */
@@ -38,7 +35,6 @@ Please visit our Website: http://www.httrack.com
/* Internal engine bytecode */
#define HTS_INTERNAL_BYTECODE
#include "htsindex.h"
#include "htsglobal.h"
#include "htslib.h"
@@ -47,7 +43,6 @@ Please visit our Website: http://www.httrack.com
#include "htshash.h"
#include "htsinthash.h"
/* Keyword Indexer Parameters */
// Maximum length for a keyword
@@ -109,33 +104,32 @@ Please visit our Website: http://www.httrack.com
/* End of Keyword Indexer Parameters */
int strcpos(char* adr,char c);
int mystrcmp(const void* _e1,const void* _e2);
int strcpos(char *adr, char c);
int mystrcmp(const void *_e1, const void *_e2);
// Global variables
int hts_index_init=1;
int hts_primindex_size=0;
FILE* fp_tmpproject=NULL;
int hts_primindex_words=0;
int hts_index_init = 1;
int hts_primindex_size = 0;
FILE *fp_tmpproject = NULL;
int hts_primindex_words = 0;
#endif
/*
Init index
*/
void index_init(const char* indexpath) {
void index_init(const char *indexpath) {
#if HTS_MAKE_KEYWORD_INDEX
#ifndef _WIN32_WCE
/* remove(concat(indexpath,"index.txt")); */
hts_index_init=1;
hts_primindex_size=0;
hts_primindex_words=0;
fp_tmpproject=tmpfile();
hts_index_init = 1;
hts_primindex_size = 0;
hts_primindex_words = 0;
fp_tmpproject = tmpfile();
#endif
#endif
}
/*
Indexing system
A little bit dirty, (quick'n dirty, in fact)
@@ -143,16 +137,19 @@ void index_init(const char* indexpath) {
Tags and javascript handled (ignored)
*/
/* Note: utf-8 */
int index_keyword(const char* html_data,LLint size,const char* mime,const char* filename,const char* indexpath) {
int index_keyword(const char *html_data, LLint size, const char *mime,
const char *filename, const char *indexpath) {
#if HTS_MAKE_KEYWORD_INDEX
char catbuff[CATBUFF_SIZE];
int intag=0,inscript=0,incomment=0;
char keyword[KEYW_LEN+32];
int i=0;
char catbuff[CATBUFF_SIZE];
int intag = 0, inscript = 0, incomment = 0;
char keyword[KEYW_LEN + 32];
int i = 0;
//
int WordIndexSize=1024;
inthash WordIndexHash=NULL;
FILE *tmpfp=NULL;
int WordIndexSize = 1024;
inthash WordIndexHash = NULL;
FILE *tmpfp = NULL;
//
// Check parameters
@@ -167,35 +164,29 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char*
// Init ?
if (hts_index_init) {
UNLINK(concat(catbuff,indexpath,"index.txt"));
UNLINK(concat(catbuff,indexpath,"sindex.html"));
hts_index_init=0;
UNLINK(concat(catbuff, indexpath, "index.txt"));
UNLINK(concat(catbuff, indexpath, "sindex.html"));
hts_index_init = 0;
}
// Check MIME type
if (is_html_mime_type(mime)) {
inscript=0;
}
inscript = 0;
}
// FIXME - temporary fix for image/svg+xml (svg)
// "IN XML" (html like, in fact :) )
else if (
(strfield2(mime,"image/svg+xml"))
||
(strfield2(mime,"image/svg-xml"))
else if ((strfield2(mime, "image/svg+xml"))
|| (strfield2(mime, "image/svg-xml"))
#if HTS_USEMMS
||
strfield2(mime,"video/x-ms-asf")
|| strfield2(mime, "video/x-ms-asf")
#endif
) {
inscript=0;
}
else if (
(strfield2(mime,"application/x-javascript"))
|| (strfield2(mime,"text/css"))
inscript = 0;
} else if ((strfield2(mime, "application/x-javascript"))
|| (strfield2(mime, "text/css"))
) {
inscript=1;
//} else if (strfield2(mime, "text/vnd.wap.wml")) { // humm won't work in many cases
// inscript=0;
inscript = 1;
//} else if (strfield2(mime, "text/vnd.wap.wml")) { // humm won't work in many cases
// inscript=0;
} else
return 0;
@@ -206,94 +197,88 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char*
// Create hash structure
// Hash tables rulez da world!
WordIndexHash=inthash_new(WordIndexSize);
WordIndexHash = inthash_new(WordIndexSize);
if (!WordIndexHash)
return 0;
// Start indexing this page
keyword[0]='\0';
while(i<size) {
if (strfield(html_data + i , "<script")) {
inscript=1;
}
else if (strfield(html_data + i , "<!--")) {
incomment=1;
}
else if (strfield(html_data + i , "</script")) {
keyword[0] = '\0';
while(i < size) {
if (strfield(html_data + i, "<script")) {
inscript = 1;
} else if (strfield(html_data + i, "<!--")) {
incomment = 1;
} else if (strfield(html_data + i, "</script")) {
if (!incomment)
inscript=0;
}
else if (strfield(html_data + i , "-->")) {
incomment=0;
}
else if (html_data[i]=='<') {
inscript = 0;
} else if (strfield(html_data + i, "-->")) {
incomment = 0;
} else if (html_data[i] == '<') {
if (!inscript)
intag=1;
}
else if (html_data[i]=='>') {
intag=0;
}
else {
intag = 1;
} else if (html_data[i] == '>') {
intag = 0;
} else {
// Okay, parse keywords
if ( (!inscript) && (!incomment) && (!intag) ) {
char cchar=html_data[i];
if ((!inscript) && (!incomment) && (!intag)) {
char cchar = html_data[i];
int pos;
int len = (int) strlen(keyword);
// Replace (ignore case, and so on..)
if ((pos=strcpos(KEYW_TRANSCODE_FROM,cchar))>=0)
cchar=KEYW_TRANSCODE_TO[pos];
if (strchr(KEYW_ACCEPT,cchar)) {
/* Ignore some characters at beginning */
if ((len>0) || (!strchr(KEYW_IGNORE_BEG,cchar))) {
keyword[len++]=cchar;
keyword[len]='\0';
}
} else if ( (strchr(KEYW_SPACE,cchar)) || (!cchar) ) {
// Replace (ignore case, and so on..)
if ((pos = strcpos(KEYW_TRANSCODE_FROM, cchar)) >= 0)
cchar = KEYW_TRANSCODE_TO[pos];
if (strchr(KEYW_ACCEPT, cchar)) {
/* Ignore some characters at beginning */
if ((len > 0) || (!strchr(KEYW_IGNORE_BEG, cchar))) {
keyword[len++] = cchar;
keyword[len] = '\0';
}
} else if ((strchr(KEYW_SPACE, cchar)) || (!cchar)) {
/* Avoid these words */
if (len>0) {
if (strchr(KEYW_NOT_BEG,keyword[0])) {
keyword[(len=0)]='\0';
if (len > 0) {
if (strchr(KEYW_NOT_BEG, keyword[0])) {
keyword[(len = 0)] = '\0';
}
}
/* Strip ending . and so */
{
int ok=0;
while((len = (int) strlen(keyword)) && (!ok)) {
if (strchr(KEYW_STRIP_END,keyword[len-1])) { /* strip it */
keyword[len-1]='\0';
} else
ok=1;
}
}
/* Store it ? */
if (len >= KEYW_MIN_LEN ) {
hts_primindex_words++;
if (inthash_inc(WordIndexHash,keyword)) { /* added new */
fprintf(tmpfp,"%s\n",keyword);
}
}
keyword[(len=0)]='\0';
} else /* Invalid */
keyword[(len=0)]='\0';
int ok = 0;
if (len>KEYW_LEN) {
keyword[(len=0)]='\0';
while((len = (int) strlen(keyword)) && (!ok)) {
if (strchr(KEYW_STRIP_END, keyword[len - 1])) { /* strip it */
keyword[len - 1] = '\0';
} else
ok = 1;
}
}
/* Store it ? */
if (len >= KEYW_MIN_LEN) {
hts_primindex_words++;
if (inthash_inc(WordIndexHash, keyword)) { /* added new */
fprintf(tmpfp, "%s\n", keyword);
}
}
keyword[(len = 0)] = '\0';
} else /* Invalid */
keyword[(len = 0)] = '\0';
if (len > KEYW_LEN) {
keyword[(len = 0)] = '\0';
}
}
}
i++;
}
// Reset temp file
fseek(tmpfp,0,SEEK_SET);
fseek(tmpfp, 0, SEEK_SET);
// Process indexing for this page
{
@@ -302,21 +287,25 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char*
if (fp_tmpproject) {
while(!feof(tmpfp)) {
char line[KEYW_LEN + 32];
linput(tmpfp,line,KEYW_LEN + 2);
linput(tmpfp, line, KEYW_LEN + 2);
if (strnotempty(line)) {
intptr_t e=0;
if (inthash_read(WordIndexHash,line,&e)) {
intptr_t e = 0;
if (inthash_read(WordIndexHash, line, &e)) {
//if (e) {
char BIGSTK savelst[HTS_URLMAXSIZE*2];
e++; /* 0 means "once" */
if (strncmp((const char*)fslash(catbuff,(char*)indexpath),filename,strlen(indexpath))==0) // couper
strcpybuff(savelst,filename+strlen(indexpath));
char BIGSTK savelst[HTS_URLMAXSIZE * 2];
e++; /* 0 means "once" */
if (strncmp((const char *) fslash(catbuff, (char *) indexpath), filename, strlen(indexpath)) == 0) // couper
strcpybuff(savelst, filename + strlen(indexpath));
else
strcpybuff(savelst,filename);
strcpybuff(savelst, filename);
// Add entry for this file and word
fprintf(fp_tmpproject,"%s %d %s\n",line,(int) (KEYW_SORT_MAXCOUNT - e),savelst);
fprintf(fp_tmpproject, "%s %d %s\n", line,
(int) (KEYW_SORT_MAXCOUNT - e), savelst);
hts_primindex_size++;
//}
}
@@ -328,7 +317,7 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char*
// Delete temp file
fclose(tmpfp);
tmpfp=NULL;
tmpfp = NULL;
// Clear hash table
inthash_delete(&WordIndexHash);
@@ -340,125 +329,134 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char*
Sort index!
*/
/* Note: NOT utf-8 */
void index_finish(const char* indexpath,int mode) {
void index_finish(const char *indexpath, int mode) {
#if HTS_MAKE_KEYWORD_INDEX
char catbuff[CATBUFF_SIZE];
char** tab;
char* blk;
char catbuff[CATBUFF_SIZE];
char **tab;
char *blk;
off_t size = fpsize(fp_tmpproject);
if (size>0) {
if (size > 0) {
//FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
if (fp_tmpproject) {
tab=(char**)malloct(sizeof(char*) * (hts_primindex_size+2) );
tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
if (tab) {
blk = malloct(size+4);
blk = malloct(size + 4);
if (blk) {
fseek(fp_tmpproject,0,SEEK_SET);
if ((INTsys)fread(blk,1,size,fp_tmpproject) == size) {
char *a=blk,*b;
int index=0;
fseek(fp_tmpproject, 0, SEEK_SET);
if ((INTsys) fread(blk, 1, size, fp_tmpproject) == size) {
char *a = blk, *b;
int index = 0;
int i;
FILE* fp;
FILE *fp;
while( (b=strchr(a,'\n')) && (index < hts_primindex_size) ) {
tab[index++]=a;
*b='\0';
a=b+1;
while((b = strchr(a, '\n')) && (index < hts_primindex_size)) {
tab[index++] = a;
*b = '\0';
a = b + 1;
}
// Sort it!
qsort(tab,index,sizeof(char*),mystrcmp);
qsort(tab, index, sizeof(char *), mystrcmp);
// Delete fp_tmpproject
fclose(fp_tmpproject);
fp_tmpproject=NULL;
fp_tmpproject = NULL;
// Write new file
if (mode == 1) // TEXT
fp=fopen(concat(catbuff,indexpath,"index.txt"),"wb");
fp = fopen(concat(catbuff, indexpath, "index.txt"), "wb");
else // HTML
fp=fopen(concat(catbuff,indexpath,"sindex.html"),"wb");
fp = fopen(concat(catbuff, indexpath, "sindex.html"), "wb");
if (fp) {
char current_word[KEYW_LEN + 32];
char word[KEYW_LEN + 32];
int hit;
int total_hit=0;
int total_line=0;
int last_pos=0;
char word0='\0';
current_word[0]='\0';
int total_hit = 0;
int total_line = 0;
int last_pos = 0;
char word0 = '\0';
if (mode == 2) { // HTML
for(i=0;i<index;i++) {
current_word[0] = '\0';
if (mode == 2) { // HTML
for(i = 0; i < index; i++) {
if (word0 != tab[i][0]) {
word0 = tab[i][0];
fprintf(fp," <a href=\"#%c\">%c</a>\r\n",word0,word0);
fprintf(fp, " <a href=\"#%c\">%c</a>\r\n", word0, word0);
}
}
word0='\0';
fprintf(fp,"<br><br>\r\n");
fprintf(fp,"<table width=\"100%%\" border=\"0\">\r\n<tr>\r\n<td>word</td>\r\n<td>location\r\n");
word0 = '\0';
fprintf(fp, "<br><br>\r\n");
fprintf(fp,
"<table width=\"100%%\" border=\"0\">\r\n<tr>\r\n<td>word</td>\r\n<td>location\r\n");
}
for(i=0;i<index;i++) {
if (sscanf(tab[i],"%s %d",word,&hit) == 2) {
char* a=strchr(tab[i],' ');
if (a) a=strchr(a+1,' ');
if (a++) { /* Yes, a++, not ++a :) */
hit=KEYW_SORT_MAXCOUNT-hit;
if (strcmp(word,current_word)) { /* New word */
for(i = 0; i < index; i++) {
if (sscanf(tab[i], "%s %d", word, &hit) == 2) {
char *a = strchr(tab[i], ' ');
if (a)
a = strchr(a + 1, ' ');
if (a++) { /* Yes, a++, not ++a :) */
hit = KEYW_SORT_MAXCOUNT - hit;
if (strcmp(word, current_word)) { /* New word */
if (total_hit) {
if (mode == 1) // TEXT
fprintf(fp,"\t=%d\r\n",total_hit);
if (mode == 1) // TEXT
fprintf(fp, "\t=%d\r\n", total_hit);
//else // HTML
// fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
if (
( ((total_hit*1000 ) / hts_primindex_words) >= KEYW_USELESS1K )
||
( ((total_line*1000) / index ) >= KEYW_USELESS1KPG )
if ((((total_hit * 1000) / hts_primindex_words) >=
KEYW_USELESS1K)
|| (((total_line * 1000) / index) >=
KEYW_USELESS1KPG)
) {
fseek(fp,last_pos,SEEK_SET);
if (mode == 1) // TEXT
fprintf(fp,"\tignored (%d)\r\n",((total_hit*1000)/hts_primindex_words));
fseek(fp, last_pos, SEEK_SET);
if (mode == 1) // TEXT
fprintf(fp, "\tignored (%d)\r\n",
((total_hit * 1000) / hts_primindex_words));
else
fprintf(fp,"(ignored) [%d hits]<br>\r\n",total_hit);
}
else {
if (mode == 1) // TEXT
fprintf(fp,"\t(%d)\r\n",((total_hit*1000)/hts_primindex_words));
fprintf(fp, "(ignored) [%d hits]<br>\r\n",
total_hit);
} else {
if (mode == 1) // TEXT
fprintf(fp, "\t(%d)\r\n",
((total_hit * 1000) / hts_primindex_words));
//else // HTML
// fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
}
}
if (mode == 1) // TEXT
fprintf(fp,"%s\r\n",word);
else { // HTML
fprintf(fp,"</td></tr>\r\n");
if (mode == 1) // TEXT
fprintf(fp, "%s\r\n", word);
else { // HTML
fprintf(fp, "</td></tr>\r\n");
if (word0 != word[0]) {
word0 = word[0];
fprintf(fp,"<th>%c</th>\r\n",word0);
fprintf(fp,"<a name=\"%c\"></a>\r\n",word0);
fprintf(fp, "<th>%c</th>\r\n", word0);
fprintf(fp, "<a name=\"%c\"></a>\r\n", word0);
}
fprintf(fp,"<tr>\r\n<td>%s</td>\r\n<td>\r\n",word);
fprintf(fp, "<tr>\r\n<td>%s</td>\r\n<td>\r\n", word);
}
fflush(fp); last_pos=ftell(fp);
strcpybuff(current_word,word);
total_hit=total_line=0;
fflush(fp);
last_pos = ftell(fp);
strcpybuff(current_word, word);
total_hit = total_line = 0;
}
total_hit+=hit;
total_hit += hit;
total_line++;
if (mode == 1) // TEXT
fprintf(fp,"\t%d %s\r\n",hit,a);
else // HTML
fprintf(fp,"<a href=\"%s\">%s</a> [%d hits]<br>\r\n",a,a,hit);
fprintf(fp, "\t%d %s\r\n", hit, a);
else // HTML
fprintf(fp, "<a href=\"%s\">%s</a> [%d hits]<br>\r\n", a,
a, hit);
}
}
}
if (mode == 2) // HTML
fprintf(fp,"</td></tr>\r\n</table>\r\n");
if (mode == 2) // HTML
fprintf(fp, "</td></tr>\r\n</table>\r\n");
fclose(fp);
}
}
freet(blk);
}
@@ -470,26 +468,26 @@ void index_finish(const char* indexpath,int mode) {
}
if (fp_tmpproject)
fclose(fp_tmpproject);
fp_tmpproject=NULL;
fp_tmpproject = NULL;
#endif
}
/* Subroutines */
#if HTS_MAKE_KEYWORD_INDEX
int strcpos(char* adr,char c) {
char* apos=strchr(adr,c);
int strcpos(char *adr, char c) {
char *apos = strchr(adr, c);
if (apos)
return (int)(apos-adr);
return (int) (apos - adr);
else
return -1;
}
int mystrcmp(const void* _e1,const void* _e2) {
char** e1=(char**)_e1;
char** e2=(char**)_e2;
return strcmp(*e1,*e2);
int mystrcmp(const void *_e1, const void *_e2) {
char **e1 = (char **) _e1;
char **e2 = (char **) _e2;
return strcmp(*e1, *e2);
}
#endif

11
src/htsindex.h
View File

@@ -17,25 +17,21 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsindex.h */
/* keyword indexing system (search index) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSKINDEX_DEFH
#define HTSKINDEX_DEFH
@@ -44,9 +40,10 @@ Please visit our Website: http://www.httrack.com
#include "htsglobal.h"
int index_keyword(const char* html_data,LLint size,const char* mime,const char* filename,const char* indexpath);
void index_init(const char* indexpath);
void index_finish(const char* indexpath,int mode);
int index_keyword(const char *html_data, LLint size, const char *mime,
const char *filename, const char *indexpath);
void index_init(const char *indexpath);
void index_finish(const char *indexpath, int mode);
#endif
#endif

242
src/htsinthash.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* hash table system (fast index) */
@@ -61,53 +58,60 @@ Please visit our Website: http://www.httrack.com
// static functions
static void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler);
static void inthash_default_free_handler(void* value);
static unsigned long int inthash_key(const char* value);
static void inthash_delchain(inthash_chain * hash,
t_inthash_freehandler free_handler);
static void inthash_default_free_handler(void *value);
static unsigned long int inthash_key(const char *value);
static void inthash_init(inthash hashtable);
// inthash -- simple hash table, using a key (char[]) and a value (ulong int)
static unsigned long int inthash_key(const char* value) {
static unsigned long int inthash_key(const char *value) {
return md5sum32(value);
}
int inthash_read_pvoid(inthash hashtable,const char* name, void** pvalue) {
int inthash_read_pvoid(inthash hashtable, const char *name, void **pvalue) {
inthash_value value = INTHASH_VALUE_NULL;
int ret = inthash_read_value(hashtable, name, (pvalue != NULL) ? &value : NULL);
int ret =
inthash_read_value(hashtable, name, (pvalue != NULL) ? &value : NULL);
if (pvalue != NULL)
*pvalue = value.ptr;
return ret;
}
int inthash_write_pvoid(inthash hashtable,const char* name, void* pvalue) {
int inthash_write_pvoid(inthash hashtable, const char *name, void *pvalue) {
inthash_value value = INTHASH_VALUE_NULL;
value.ptr = pvalue;
return inthash_write_value(hashtable, name, value);
}
void inthash_add_pvoid(inthash hashtable, const char* name, void* pvalue) {
void inthash_add_pvoid(inthash hashtable, const char *name, void *pvalue) {
inthash_value value = INTHASH_VALUE_NULL;
value.ptr = pvalue;
inthash_add_value(hashtable, name, value);
}
// Check for duplicate entry (==1 : added)
int inthash_write(inthash hashtable,const char* name,intptr_t intvalue) {
int inthash_write(inthash hashtable, const char *name, intptr_t intvalue) {
inthash_value value = INTHASH_VALUE_NULL;
value.intg = intvalue;
return inthash_write_value(hashtable, name, value);
}
int inthash_write_value(inthash hashtable,const char* name,inthash_value value) {
int inthash_write_value(inthash hashtable, const char *name,
inthash_value value) {
int pos = (inthash_key(name) % hashtable->hash_size);
inthash_chain* h=hashtable->hash[pos];
while (h) {
if (strcmp(h->name,name)==0) {
inthash_chain *h = hashtable->hash[pos];
while(h) {
if (strcmp(h->name, name) == 0) {
/* Delete element */
if (hashtable->flag_valueismalloc) {
void* ptr = h->value.ptr;
void *ptr = h->value.ptr;
if (ptr != NULL) {
if (hashtable->free_handler)
hashtable->free_handler(ptr);
@@ -116,147 +120,147 @@ int inthash_write_value(inthash hashtable,const char* name,inthash_value value)
}
}
/* Insert */
h->value=value;
h->value = value;
return 0;
}
h=h->next;
h = h->next;
}
// Not found, add it!
inthash_add_value(hashtable,name,value);
inthash_add_value(hashtable, name, value);
return 1;
}
// Increment pos value, create one if necessary (=0)
// (==1 : created)
int inthash_inc(inthash hashtable,const char* name) {
intptr_t value=0;
int r=0;
if (inthash_read(hashtable,name,&value)) {
int inthash_inc(inthash hashtable, const char *name) {
intptr_t value = 0;
int r = 0;
if (inthash_read(hashtable, name, &value)) {
value++;
} else { /* create new value */
value = 0;
r = 1;
}
else { /* create new value */
value=0;
r=1;
}
inthash_write(hashtable,name,value);
inthash_write(hashtable, name, value);
return (r);
}
// Does not check for duplicate entry
void inthash_add(inthash hashtable, const char* name, intptr_t intvalue) {
void inthash_add(inthash hashtable, const char *name, intptr_t intvalue) {
inthash_value value = INTHASH_VALUE_NULL;
memset(&value, 0, sizeof(value));
value.intg = intvalue;
inthash_add_value(hashtable, name, value);
}
void inthash_add_value(inthash hashtable, const char* name, inthash_value value) {
void inthash_add_value(inthash hashtable, const char *name, inthash_value value) {
int pos = (inthash_key(name) % hashtable->hash_size);
inthash_chain** h=&hashtable->hash[pos];
inthash_chain **h = &hashtable->hash[pos];
while (*h)
h=&((*h)->next);
*h=(inthash_chain*)calloct(1,
sizeof(inthash_chain)
+
strlen(name)+2
);
while(*h)
h = &((*h)->next);
*h = (inthash_chain *) calloct(1, sizeof(inthash_chain)
+ strlen(name) + 2);
if (*h) {
(*h)->name=((char*)(*h)) + sizeof(inthash_chain);
(*h)->next=NULL;
strcpybuff((*h)->name,name);
(*h)->value=value;
(*h)->name = ((char *) (*h)) + sizeof(inthash_chain);
(*h)->next = NULL;
strcpybuff((*h)->name, name);
(*h)->value = value;
hashtable->nitems++;
}
}
void* inthash_addblk(inthash hashtable,const char* name,int blksize) {
void *inthash_addblk(inthash hashtable, const char *name, int blksize) {
int pos = (inthash_key(name) % hashtable->hash_size);
inthash_chain** h=&hashtable->hash[pos];
inthash_chain **h = &hashtable->hash[pos];
while (*h)
h=&((*h)->next);
*h=(inthash_chain*)calloct(1,
sizeof(inthash_chain)
+
strlen(name)+2
+
blksize
);
while(*h)
h = &((*h)->next);
*h = (inthash_chain *) calloct(1, sizeof(inthash_chain)
+ strlen(name) + 2 + blksize);
if (*h) {
(*h)->name = ((char*)(*h)) + sizeof(inthash_chain);
(*h)->next=NULL;
strcpybuff((*h)->name,name);
(*h)->value.ptr = (void*) ( ((char*)(*h)) + sizeof(inthash_chain) + strlen(name) + 2 );
(*h)->name = ((char *) (*h)) + sizeof(inthash_chain);
(*h)->next = NULL;
strcpybuff((*h)->name, name);
(*h)->value.ptr =
(void *) (((char *) (*h)) + sizeof(inthash_chain) + strlen(name) + 2);
hashtable->nitems++;
return (*h)->value.ptr;
}
return NULL;
}
int inthash_read(inthash hashtable,const char* name,intptr_t* intvalue) {
int inthash_read(inthash hashtable, const char *name, intptr_t * intvalue) {
inthash_value value = INTHASH_VALUE_NULL;
int ret = inthash_read_value(hashtable, name, (intvalue != NULL) ? &value : NULL);
int ret =
inthash_read_value(hashtable, name, (intvalue != NULL) ? &value : NULL);
if (intvalue != NULL)
*intvalue = value.intg;
return ret;
}
int inthash_read_value(inthash hashtable,const char* name,inthash_value* value) {
int inthash_read_value(inthash hashtable, const char *name,
inthash_value * value) {
int pos = (inthash_key(name) % hashtable->hash_size);
inthash_chain* h=hashtable->hash[pos];
while (h) {
if (strcmp(h->name,name)==0) {
inthash_chain *h = hashtable->hash[pos];
while(h) {
if (strcmp(h->name, name) == 0) {
if (value != NULL)
*value=h->value;
*value = h->value;
return 1;
}
h=h->next;
h = h->next;
}
return 0;
}
int inthash_exists(inthash hashtable, const char* name) {
int inthash_exists(inthash hashtable, const char *name) {
return inthash_read_value(hashtable, name, NULL);
}
int inthash_remove(inthash hashtable,const char* name) {
int inthash_remove(inthash hashtable, const char *name) {
int pos = (inthash_key(name) % hashtable->hash_size);
inthash_chain** h=&hashtable->hash[pos];
t_inthash_freehandler free_handler=NULL;
if ( hashtable->flag_valueismalloc ) {
if ( hashtable->free_handler )
free_handler=hashtable->free_handler;
inthash_chain **h = &hashtable->hash[pos];
t_inthash_freehandler free_handler = NULL;
if (hashtable->flag_valueismalloc) {
if (hashtable->free_handler)
free_handler = hashtable->free_handler;
else
free_handler=inthash_default_free_handler;
free_handler = inthash_default_free_handler;
}
while (*h) {
if (strcmp((*h)->name,name)==0) {
inthash_chain* next;
while(*h) {
if (strcmp((*h)->name, name) == 0) {
inthash_chain *next;
if (free_handler) {
if ((*h)->value.ptr) {
void* ptr = (*h)->value.ptr;
void *ptr = (*h)->value.ptr;
if (free_handler)
free_handler(ptr);
else
freet(ptr);
(*h)->value.ptr=0;
(*h)->value.ptr = 0;
}
}
next=(*h)->next;
next = (*h)->next;
freet(*h);
*h=next;
*h = next;
hashtable->nitems--;
return 1;
}
h=&((*h)->next);
h = &((*h)->next);
}
return 0;
}
int inthash_readptr(inthash hashtable,const char* name,intptr_t* value) {
int inthash_readptr(inthash hashtable, const char *name, intptr_t * value) {
int ret;
*value = 0;
ret = inthash_read(hashtable, name, value);
if (*value == 0)
@@ -266,30 +270,34 @@ int inthash_readptr(inthash hashtable,const char* name,intptr_t* value) {
static void inthash_init(inthash hashtable) {
unsigned int i;
for(i=0;i<hashtable->hash_size;i++) {
hashtable->hash[i]=NULL;
for(i = 0; i < hashtable->hash_size; i++) {
hashtable->hash[i] = NULL;
}
}
static void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler) {
static void inthash_delchain(inthash_chain * hash,
t_inthash_freehandler free_handler) {
while(hash != NULL) {
inthash_chain* next=hash->next;
if (free_handler) { // pos is a malloc() block, delete it!
inthash_chain *next = hash->next;
if (free_handler) { // pos is a malloc() block, delete it!
if (hash->value.ptr) {
void* ptr = hash->value.ptr;
void *ptr = hash->value.ptr;
if (free_handler)
free_handler(ptr);
else
freet(ptr);
hash->value.ptr=0;
hash->value.ptr = 0;
}
}
freet(hash);
hash=next;
hash = next;
}
}
static void inthash_default_free_handler(void* value) {
static void inthash_default_free_handler(void *value) {
if (value)
freet(value);
}
@@ -297,12 +305,14 @@ static void inthash_default_free_handler(void* value) {
// --
inthash inthash_new(int size) {
inthash hashtable=(inthash)calloct(1,sizeof(struct_inthash));
inthash hashtable = (inthash) calloct(1, sizeof(struct_inthash));
if (hashtable) {
hashtable->hash_size=0;
hashtable->flag_valueismalloc=0;
if ((hashtable->hash=(inthash_chain**)calloct(size,sizeof(inthash_chain*)))) {
hashtable->hash_size=size;
hashtable->hash_size = 0;
hashtable->flag_valueismalloc = 0;
if ((hashtable->hash =
(inthash_chain **) calloct(size, sizeof(inthash_chain *)))) {
hashtable->hash_size = size;
inthash_init(hashtable);
}
hashtable->nitems = 0;
@@ -317,41 +327,43 @@ int inthash_created(inthash hashtable) {
return 0;
}
void inthash_value_is_malloc(inthash hashtable,int flag) {
hashtable->flag_valueismalloc=flag;
void inthash_value_is_malloc(inthash hashtable, int flag) {
hashtable->flag_valueismalloc = flag;
}
void inthash_value_set_free_handler(inthash hashtable, t_inthash_freehandler free_handler) {
void inthash_value_set_free_handler(inthash hashtable,
t_inthash_freehandler free_handler) {
hashtable->free_handler = free_handler;
}
unsigned int inthash_nitems(inthash hashtable) {
if (hashtable!= NULL)
if (hashtable != NULL)
return hashtable->nitems;
return 0;
}
void inthash_delete(inthash* hashtable) {
void inthash_delete(inthash * hashtable) {
if (hashtable) {
if (*hashtable) {
if ((*hashtable)->hash) {
unsigned int i;
t_inthash_freehandler free_handler=NULL;
if ( (*hashtable)->flag_valueismalloc ) {
if ( (*hashtable)->free_handler )
free_handler=(*hashtable)->free_handler;
t_inthash_freehandler free_handler = NULL;
if ((*hashtable)->flag_valueismalloc) {
if ((*hashtable)->free_handler)
free_handler = (*hashtable)->free_handler;
else
free_handler=inthash_default_free_handler;
free_handler = inthash_default_free_handler;
}
for(i=0;i<(*hashtable)->hash_size;i++) {
for(i = 0; i < (*hashtable)->hash_size; i++) {
inthash_delchain((*hashtable)->hash[i], free_handler);
(*hashtable)->hash[i]=NULL;
(*hashtable)->hash[i] = NULL;
}
freet((*hashtable)->hash);
(*hashtable)->hash = NULL;
}
freet(*hashtable);
*hashtable=NULL;
*hashtable = NULL;
}
}
}
@@ -360,6 +372,7 @@ void inthash_delete(inthash* hashtable) {
struct_inthash_enum inthash_enum_new(inthash hashtable) {
struct_inthash_enum e;
memset(&e, 0, sizeof(e));
e.index = 0;
e.item = NULL;
@@ -367,8 +380,9 @@ struct_inthash_enum inthash_enum_new(inthash hashtable) {
return e;
}
inthash_chain* inthash_enum_next(struct_inthash_enum* e) {
inthash_chain* item = NULL;
inthash_chain *inthash_enum_next(struct_inthash_enum * e) {
inthash_chain *item = NULL;
if (e != NULL) {
while(e->item == NULL && e->index < (int) e->table->hash_size) {
e->item = e->table->hash[e->index];

75
src/htsinthash.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* hash table system (fast index) */
@@ -38,7 +35,7 @@ Please visit our Website: http://www.httrack.com
// inthash -- simple hash table, using a key (char[]) and a value (uintptr_t)
#ifndef HTSINTHASH_DEFH
#define HTSINTHASH_DEFH
#define HTSINTHASH_DEFH
/* Includes */
#ifdef _WIN32
@@ -52,8 +49,8 @@ Please visit our Website: http://www.httrack.com
// value
typedef union inthash_value {
uintptr_t intg; /* integer value */
void* ptr; /* ptr value */
uintptr_t intg; /* integer value */
void *ptr; /* ptr value */
} inthash_value;
#define INTHASH_VALUE_NULL { 0 }
@@ -64,12 +61,12 @@ typedef union inthash_value {
typedef struct inthash_chain inthash_chain;
#endif
struct inthash_chain {
char* name; /* key (name) */
inthash_value value; /* value */
struct inthash_chain* next; /* next element */
char *name; /* key (name) */
inthash_value value; /* value */
struct inthash_chain *next; /* next element */
};
typedef void (* t_inthash_freehandler)(void* value);
typedef void (*t_inthash_freehandler) (void *value);
/* inthash structure */
#ifndef HTS_DEF_FWSTRUCT_struct_inthash
@@ -77,7 +74,7 @@ typedef void (* t_inthash_freehandler)(void* value);
typedef struct struct_inthash struct_inthash, *inthash;
#endif
struct struct_inthash {
inthash_chain** hash;
inthash_chain **hash;
unsigned int nitems;
t_inthash_freehandler free_handler;
unsigned int hash_size;
@@ -92,7 +89,7 @@ typedef struct struct_inthash_enum struct_inthash_enum;
struct struct_inthash_enum {
inthash table;
int index;
inthash_chain* item;
inthash_chain *item;
};
/* Library internal definictions */
@@ -101,34 +98,42 @@ struct struct_inthash_enum {
// main functions:
/* Hash functions: */
inthash inthash_new(int size); /* Create a new hash table */
int inthash_created(inthash hashtable); /* Test if the hash table was successfully created */
unsigned int inthash_nitems(inthash hashtable); /* Number of items */
void inthash_delete(inthash* hashtable); /* Delete an hash table */
void inthash_value_is_malloc(inthash hashtable,int flag); /* Is the 'value' member a value that needs to be free()'ed ? */
void inthash_value_set_free_handler(inthash hashtable, /* value free() handler (default one is 'free') */
t_inthash_freehandler free_handler);
inthash inthash_new(int size); /* Create a new hash table */
int inthash_created(inthash hashtable); /* Test if the hash table was successfully created */
unsigned int inthash_nitems(inthash hashtable); /* Number of items */
void inthash_delete(inthash * hashtable); /* Delete an hash table */
void inthash_value_is_malloc(inthash hashtable, int flag); /* Is the 'value' member a value that needs to be free()'ed ? */
void inthash_value_set_free_handler(inthash hashtable, /* value free() handler (default one is 'free') */
t_inthash_freehandler free_handler);
/* */
int inthash_read(inthash hashtable,const char* name,intptr_t* intvalue); /* Read entry from the hash table */
int inthash_readptr(inthash hashtable,const char* name,intptr_t* intvalue); /* Same function, but returns 0 upon null ptr */
int inthash_exists(inthash hashtable, const char* name); /* Is the key existing ? */
int inthash_read(inthash hashtable, const char *name, intptr_t * intvalue); /* Read entry from the hash table */
int inthash_readptr(inthash hashtable, const char *name, intptr_t * intvalue); /* Same function, but returns 0 upon null ptr */
int inthash_exists(inthash hashtable, const char *name); /* Is the key existing ? */
/* */
int inthash_read_value(inthash hashtable,const char* name,inthash_value* value);
int inthash_write_value(inthash hashtable,const char* name,inthash_value value);
void inthash_add_value(inthash hashtable, const char* name, inthash_value value);
int inthash_read_value(inthash hashtable, const char *name,
inthash_value * value);
int inthash_write_value(inthash hashtable, const char *name,
inthash_value value);
void inthash_add_value(inthash hashtable, const char *name,
inthash_value value);
/* */
int inthash_read_pvoid(inthash hashtable,const char* name, void** value);
int inthash_write_pvoid(inthash hashtable,const char* name, void* value);
void inthash_add_pvoid(inthash hashtable, const char* name, void* value);
int inthash_read_pvoid(inthash hashtable, const char *name, void **value);
int inthash_write_pvoid(inthash hashtable, const char *name, void *value);
void inthash_add_pvoid(inthash hashtable, const char *name, void *value);
/* */
void inthash_add(inthash hashtable,const char* name,intptr_t value); /* Add entry in the hash table */
void* inthash_addblk(inthash hashtable,const char* name,int blksize); /* Add entry in the hash table and set value to a new memory block */
int inthash_write(inthash hashtable,const char* name,intptr_t value); /* Overwrite/add entry in the hash table */
int inthash_inc(inthash hashtable,const char* name); /* Increment entry in the hash table */
int inthash_remove(inthash hashtable,const char* name); /* Remove an entry from the hashtable */
void inthash_add(inthash hashtable, const char *name, intptr_t value); /* Add entry in the hash table */
void *inthash_addblk(inthash hashtable, const char *name, int blksize); /* Add entry in the hash table and set value to a new memory block */
int inthash_write(inthash hashtable, const char *name, intptr_t value); /* Overwrite/add entry in the hash table */
int inthash_inc(inthash hashtable, const char *name); /* Increment entry in the hash table */
int inthash_remove(inthash hashtable, const char *name); /* Remove an entry from the hashtable */
/* */
struct_inthash_enum inthash_enum_new(inthash hashtable); /* Start a new enumerator */
inthash_chain* inthash_enum_next(struct_inthash_enum* e); /* Fetch an item in the enumerator */
struct_inthash_enum inthash_enum_new(inthash hashtable); /* Start a new enumerator */
inthash_chain *inthash_enum_next(struct_inthash_enum * e); /* Fetch an item in the enumerator */
/* End of hash functions: */
#endif

347
src/htsjava.c
View File

@@ -17,24 +17,20 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Java classes parser */
/* Author: Yann Philippot */
/* ------------------------------------------------------------ */
/* Version: Oct/2000 */
/* Fixed: problems with class structure (10/2000) */
@@ -69,8 +65,9 @@ Please visit our Website: http://www.httrack.com
#include "htsjava.h"
static int reverse_endian(void) {
int endian = 1;
return ( * ( (char*) &endian) == 1);
int endian = 1;
return (*((char *) &endian) == 1);
}
/* big/little endian swap */
@@ -78,10 +75,11 @@ static int reverse_endian(void) {
#define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) )
/* Static definitions */
static RESP_STRUCT readtable(htsmoduleStruct* str,FILE *fp,RESP_STRUCT,int*);
static unsigned short int readshort(FILE *fp);
static int tris(httrackp *opt,char*);
static char * printname(char [1024],char [1024]);
static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT,
int *);
static unsigned short int readshort(FILE * fp);
static int tris(httrackp * opt, char *);
static char *printname(char[1024], char[1024]);
// ** HTS_xx sinon pas pris par VC++
#define HTS_CLASS 7
@@ -106,38 +104,41 @@ static const char *libName = "htsjava";
#define strncasecmp(a,b,n) strnicmp(a,b,n)
#endif
static int detect_mime(htsmoduleStruct* str) {
const char* savename = str->filename;
static int detect_mime(htsmoduleStruct * str) {
const char *savename = str->filename;
if (savename) {
int len = (int) strlen(savename);
if (len > 6 && strcasecmp(savename + len - 6,".class") == 0) {
if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) {
return 1;
}
}
return 0;
}
static int hts_detect_java(t_hts_callbackarg *carg, httrackp *opt,
htsmoduleStruct* str)
{
static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt,
htsmoduleStruct * str) {
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) {
if (CALLBACKARG_PREV_FUN(carg, detect)(CALLBACKARG_PREV_CARG(carg), opt, str)) {
return 1; /* Found before us, let them have the priority */
if (CALLBACKARG_PREV_FUN(carg, detect)
(CALLBACKARG_PREV_CARG(carg), opt, str)) {
return 1; /* Found before us, let them have the priority */
}
}
/* Check MIME */
if (detect_mime(str)) {
str->wrapper_name = libName; /* Our ID */
return 1; /* Known format, we take it */
str->wrapper_name = libName; /* Our ID */
return 1; /* Known format, we take it */
}
return 0; /* Unknown format */
return 0; /* Unknown format */
}
static off_t fsize(const char* s) {
static off_t fsize(const char *s) {
STRUCT_STAT st;
if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
return st.st_size;
} else {
@@ -145,17 +146,18 @@ static off_t fsize(const char* s) {
}
}
static int hts_parse_java(t_hts_callbackarg *carg, httrackp *opt,
htsmoduleStruct* str)
{
static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
htsmoduleStruct * str) {
/* The wrapper_name memebr has changed: not for us anymore */
if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) {
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) {
return CALLBACKARG_PREV_FUN(carg, parse)(CALLBACKARG_PREV_CARG(carg), opt, str);
return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg),
opt, str);
}
strcpy(str->err_msg, "unexpected error: bad wrapper_name and no previous wrapper");
return 0; /* Unexpected error */
strcpy(str->err_msg,
"unexpected error: bad wrapper_name and no previous wrapper");
return 0; /* Unexpected error */
} else {
if (detect_mime(str)) {
@@ -164,74 +166,82 @@ static int hts_parse_java(t_hts_callbackarg *carg, httrackp *opt,
FILE *fpout;
JAVA_HEADER header;
RESP_STRUCT *tab;
const char* file = str->filename;
const char *file = str->filename;
str->relativeToHtmlLink = 1;
#if JAVADEBUG
printf("fopen\n");
#endif
if ((fpout = FOPEN(fconv(catbuff, file), "r+b")) == NULL)
{
if ((fpout = FOPEN(fconv(catbuff, file), "r+b")) == NULL) {
//fprintf(stderr, "Cannot open input file.\n");
sprintf(str->err_msg,"Unable to open file %s",file);
return 0; // une erreur..
sprintf(str->err_msg, "Unable to open file %s", file);
return 0; // une erreur..
}
#if JAVADEBUG
printf("fread\n");
#endif
//if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) { // pas complet..
if (fread(&header,1,10,fpout) != 10) { // pas complet..
if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
fclose(fpout);
sprintf(str->err_msg,"File header too small (file len = "LLintP")",(LLint)fsize(file));
sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
(LLint) fsize(file));
return 0;
}
#if JAVADEBUG
printf("header\n");
#endif
// tester en tête
if (reverse_endian()) {
header.magic = hts_swap32(header.magic);
header.count = hts_swap16(header.count);
header.count = hts_swap16(header.count);
}
if(header.magic!=0xCAFEBABE) {
sprintf(str->err_msg,"non java file");
if (fpout) { fclose(fpout); fpout=NULL; }
if (header.magic != 0xCAFEBABE) {
sprintf(str->err_msg, "non java file");
if (fpout) {
fclose(fpout);
fpout = NULL;
}
return 0;
}
tab =(RESP_STRUCT*)calloc(header.count,sizeof(RESP_STRUCT));
tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
if (!tab) {
sprintf(str->err_msg,"Unable to alloc %d bytes",(int)sizeof(RESP_STRUCT));
if (fpout) { fclose(fpout); fpout=NULL; }
return 0; // erreur..
sprintf(str->err_msg, "Unable to alloc %d bytes",
(int) sizeof(RESP_STRUCT));
if (fpout) {
fclose(fpout);
fpout = NULL;
}
return 0; // erreur..
}
#if JAVADEBUG
printf("calchead\n");
#endif
{
int i;
for (i = 1; i < header.count; i++) {
int err=0; // ++
tab[i]=readtable(str,fpout,tab[i],&err);
for(i = 1; i < header.count; i++) {
int err = 0; // ++
tab[i] = readtable(str, fpout, tab[i], &err);
if (!err) {
if ((tab[i].type == HTS_LONG) ||(tab[i].type == HTS_DOUBLE)) i++; //2 element si double ou float
} else { // ++ une erreur est survenue!
if (strnotempty(str->err_msg)==0)
strcpy(str->err_msg,"Internal readtable error");
if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE))
i++; //2 element si double ou float
} else { // ++ une erreur est survenue!
if (strnotempty(str->err_msg) == 0)
strcpy(str->err_msg, "Internal readtable error");
free(tab);
if (fpout) { fclose(fpout); fpout=NULL; }
if (fpout) {
fclose(fpout);
fpout = NULL;
}
return 0;
}
}
}
#if JAVADEBUG
printf("addfiles\n");
#endif
@@ -240,251 +250,266 @@ static int hts_parse_java(t_hts_callbackarg *carg, httrackp *opt,
unsigned int Class;
unsigned int SClass;
int i;
//acess = readshort(fpout);
Class = readshort(fpout);
SClass = readshort(fpout);
for (i = 1; i <header.count; i++) {
for(i = 1; i < header.count; i++) {
if (tab[i].type == HTS_CLASS) {
if ((tab[i].index1<header.count) && (tab[i].index1>=0)) {
if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) {
if ((tab[i].index1 != SClass) && (tab[i].index1 != Class)
&& (tab[tab[i].index1].name[0] != '[')) {
if((tab[i].index1!=SClass) && (tab[i].index1!=Class) && (tab[tab[i].index1].name[0]!='[')) {
if(!strstr(tab[tab[i].index1].name,"java/")) {
if (!strstr(tab[tab[i].index1].name, "java/")) {
char BIGSTK tempo[1024];
tempo[0]='\0';
sprintf(tempo,"%s.class",tab[tab[i].index1].name);
tempo[0] = '\0';
sprintf(tempo, "%s.class", tab[tab[i].index1].name);
#if JAVADEBUG
printf("add %s\n",tempo);
printf("add %s\n", tempo);
#endif
if (tab[tab[i].index1].file_position >= 0)
str->addLink(str,tempo); /* tab[tab[i].index1].file_position */
str->addLink(str, tempo); /* tab[tab[i].index1].file_position */
}
}
} else {
i=header.count; // exit
} else {
i = header.count; // exit
}
}
}
}
#if JAVADEBUG
printf("end\n");
#endif
free(tab);
if (fpout) { fclose(fpout); fpout=NULL; }
if (fpout) {
fclose(fpout);
fpout = NULL;
}
return 1;
} else {
strcpy(str->err_msg, "bad MIME type");
}
}
return 0; /* Error */
return 0; /* Error */
}
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
/* Plug callback functions */
CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL);
CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL);
return 1; /* success */
return 1; /* success */
}
// error: !=0 si erreur fatale
static RESP_STRUCT readtable(htsmoduleStruct* str,
FILE *fp, RESP_STRUCT trans, int* error)
{
char rname[1024];
static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
RESP_STRUCT trans, int *error) {
char rname[1024];
unsigned short int length;
int j;
*error = 0; // pas d'erreur
trans.file_position=-1;
trans.type = (int)(unsigned char)fgetc(fp);
*error = 0; // pas d'erreur
trans.file_position = -1;
trans.type = (int) (unsigned char) fgetc(fp);
switch (trans.type) {
case HTS_CLASS:
strcpy(trans.name,"Class");
strcpy(trans.name, "Class");
trans.index1 = readshort(fp);
break;
case HTS_FIELDREF:
strcpy(trans.name,"Field Reference");
strcpy(trans.name, "Field Reference");
trans.index1 = readshort(fp);
readshort(fp);
break;
case HTS_METHODREF:
strcpy(trans.name,"Method Reference");
strcpy(trans.name, "Method Reference");
trans.index1 = readshort(fp);
readshort(fp);
break;
case HTS_INTERFACE:
strcpy(trans.name,"Interface Method Reference");
trans.index1 =readshort(fp);
strcpy(trans.name, "Interface Method Reference");
trans.index1 = readshort(fp);
readshort(fp);
break;
case HTS_NAMEANDTYPE:
strcpy(trans.name,"Name and Type");
strcpy(trans.name, "Name and Type");
trans.index1 = readshort(fp);
readshort(fp);
break;
case HTS_STRING: // CONSTANT_String
strcpy(trans.name,"String");
case HTS_STRING: // CONSTANT_String
strcpy(trans.name, "String");
trans.index1 = readshort(fp);
break;
case HTS_INTEGER:
strcpy(trans.name,"Integer");
for(j=0;j<4;j++) fgetc(fp);
strcpy(trans.name, "Integer");
for(j = 0; j < 4; j++)
fgetc(fp);
break;
case HTS_FLOAT:
strcpy(trans.name,"Float");
for(j=0;j<4;j++) fgetc(fp);
strcpy(trans.name, "Float");
for(j = 0; j < 4; j++)
fgetc(fp);
break;
case HTS_LONG:
strcpy(trans.name,"Long");
for(j=0;j<8;j++) fgetc(fp);
strcpy(trans.name, "Long");
for(j = 0; j < 8; j++)
fgetc(fp);
break;
case HTS_DOUBLE:
strcpy(trans.name,"Double");
for(j=0;j<8;j++) fgetc(fp);
strcpy(trans.name, "Double");
for(j = 0; j < 8; j++)
fgetc(fp);
break;
case HTS_ASCIZ:
case HTS_UNICODE:
if (trans.type == HTS_ASCIZ)
strcpy(trans.name,"HTS_ASCIZ");
strcpy(trans.name, "HTS_ASCIZ");
else
strcpy(trans.name,"HTS_UNICODE");
strcpy(trans.name, "HTS_UNICODE");
{
char BIGSTK buffer[1024];
char BIGSTK buffer[1024];
char *p;
p=&buffer[0];
p = &buffer[0];
//fflush(fp);
trans.file_position=ftell(fp);
trans.file_position = ftell(fp);
length = readshort(fp);
if (length<HTS_URLMAXSIZE) {
if (length < HTS_URLMAXSIZE) {
// while ((length > 0) && (length<500)) {
while (length > 0) {
*p++ =fgetc(fp);
while(length > 0) {
*p++ = fgetc(fp);
length--;
}
*p='\0';
*p = '\0';
//#if JDEBUG
// if(tris(buffer)==1) printf("%s\n ",buffer);
// if(tris(buffer)==2) printf("%s\n ",printname(buffer));
//#endif
if(tris(str->opt,buffer)==1) str->addLink(str, buffer); /* trans.file_position */
else if(tris(str->opt,buffer)==2) str->addLink(str, printname(rname,buffer));
if (tris(str->opt, buffer) == 1)
str->addLink(str, buffer); /* trans.file_position */
else if (tris(str->opt, buffer) == 2)
str->addLink(str, printname(rname, buffer));
strcpy(trans.name,buffer);
} else { // gros pb
while ( (length > 0) && (!feof(fp))) {
strcpy(trans.name, buffer);
} else { // gros pb
while((length > 0) && (!feof(fp))) {
fgetc(fp);
length--;
}
if (!feof(fp)) {
trans.type=-1;
trans.type = -1;
} else {
sprintf(str->err_msg,"Internal stucture error (ASCII)");
sprintf(str->err_msg, "Internal stucture error (ASCII)");
*error = 1;
}
return(trans);
return (trans);
}
}
break;
default:
// printf("Type inconnue\n");
// on arrête tout
sprintf(str->err_msg,"Internal structure unknown (type %d)",trans.type);
sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
*error = 1;
return(trans);
return (trans);
break;
}
return(trans);
}
return (trans);
}
static unsigned short int readshort(FILE *fp)
{
static unsigned short int readshort(FILE * fp) {
unsigned short int valint;
fread(&valint,sizeof(valint),1,fp);
fread(&valint, sizeof(valint), 1, fp);
if (reverse_endian())
return hts_swap16(valint);
else
return valint;
}
static int tris(httrackp *opt,char * buffer)
{
char catbuff[CATBUFF_SIZE];
static int tris(httrackp * opt, char *buffer) {
char catbuff[CATBUFF_SIZE];
//
// Java
if((buffer[0]=='[') && buffer[1]=='L' && (!strstr(buffer,"java/")) )
if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/")))
return 2;
if (strstr(buffer,".gif") || strstr(buffer,".jpg") || strstr(buffer,".jpeg") || strstr(buffer,".au") )
if (strstr(buffer, ".gif") || strstr(buffer, ".jpg")
|| strstr(buffer, ".jpeg") || strstr(buffer, ".au"))
return 1;
// Ajouts R.X: test type
// Autres fichiers
{
char type[256];
type[0]='\0';
get_httptype(opt,type,buffer,0);
if (strnotempty(type)) // type reconnu!
type[0] = '\0';
get_httptype(opt, type, buffer, 0);
if (strnotempty(type)) // type reconnu!
return 1;
// ajout RX 05/2001
else if (is_dyntype(get_ext(catbuff, buffer))) // asp,cgi...
else if (is_dyntype(get_ext(catbuff, buffer))) // asp,cgi...
return 1;
}
return 0;
}
static char * printname(char rname[1024], char name[1024])
{
static char *printname(char rname[1024], char name[1024]) {
char *p;
char *p1;
int j;
rname[0]='\0';
rname[0] = '\0';
//
p=&name[0];
if(*p!='[') return "";
p+=2;
p = &name[0];
if (*p != '[')
return "";
p += 2;
//rname=(char*)calloct(strlen(name)+8,sizeof(char));
p1=rname;
for (j = 0; j < (int) strlen(name); j++,p++) {
if (*p == '/') *p1='.';
if (*p==';'){*p1='\0';
strcat(rname,".class");
return (rname);}
else *p1=*p;
p1 = rname;
for(j = 0; j < (int) strlen(name); j++, p++) {
if (*p == '/')
*p1 = '.';
if (*p == ';') {
*p1 = '\0';
strcat(rname, ".class");
return (rname);
} else
*p1 = *p;
p1++;
}
p1-=3;
*p1='\0';
p1 -= 3;
*p1 = '\0';
return (rname);
}

11
src/htsjava.h
View File

@@ -17,33 +17,29 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Java classes parser .h */
/* Author: Yann Philippot */
/* ------------------------------------------------------------ */
#ifndef HTSJAVA_DEFH
#define HTSJAVA_DEFH
#define HTSJAVA_DEFH
#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
#define HTS_DEF_FWSTRUCT_JAVA_HEADER
typedef struct JAVA_HEADER JAVA_HEADER;
#endif
struct JAVA_HEADER {
unsigned long int magic;
unsigned long int magic;
unsigned short int minor;
unsigned short int major;
unsigned short int count;
@@ -61,11 +57,10 @@ struct RESP_STRUCT {
char name[1024];
};
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
EXTERNAL_FUNCTION int hts_plug_java(httrackp *opt, const char* argv);
EXTERNAL_FUNCTION int hts_plug_java(httrackp * opt, const char *argv);
#endif

5740
src/htslib.c
View File

File diff suppressed because it is too large Load Diff

581
src/htslib.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Subroutines .h */
/* Author: Xavier Roche */
@@ -37,7 +34,7 @@ Please visit our Website: http://www.httrack.com
// Fichier librairie .h
#ifndef HTS_DEFH
#define HTS_DEFH
#define HTS_DEFH
/* Forward definitions */
#ifndef HTS_DEF_FWSTRUCT_htsrequest
@@ -90,10 +87,11 @@ typedef struct t_dnscache t_dnscache;
#else
#define MSVC2003INLINEBUG
#endif
MSVC2003INLINEBUG HTS_STATIC char* getHtsOptBuff_(httrackp *opt) {
opt->state.concat.index = ( opt->state.concat.index + 1 ) % 16;
return opt->state.concat.buff[opt->state.concat.index];
MSVC2003INLINEBUG HTS_STATIC char *getHtsOptBuff_(httrackp * opt) {
opt->state.concat.index = (opt->state.concat.index + 1) % 16;
return opt->state.concat.buff[opt->state.concat.index];
}
#undef MSVC2003INLINEBUG
#define OPT_GET_BUFF(OPT) ( getHtsOptBuff_(OPT) )
@@ -106,76 +104,75 @@ struct htsrequest_proxy {
int active;
char name[1024];
int port;
char bindhost[256]; // bind this host
};
char bindhost[256]; // bind this host
};
#ifndef HTS_DEF_FWSTRUCT_htsrequest
#define HTS_DEF_FWSTRUCT_htsrequest
typedef struct htsrequest htsrequest;
#endif
struct htsrequest {
short int user_agent_send; // user agent (ex: httrack/1.0 [sun])
short int http11; // l'en tête peut (doit) être signé HTTP/1.1 et non HTTP/1.0
short int nokeepalive; // pas de keep-alive
short int range_used; // Range utilisé
short int nocompression; // Pas de compression
short int flush_garbage; // recycled
short int user_agent_send; // user agent (ex: httrack/1.0 [sun])
short int http11; // l'en tête peut (doit) être signé HTTP/1.1 et non HTTP/1.0
short int nokeepalive; // pas de keep-alive
short int range_used; // Range utilisé
short int nocompression; // Pas de compression
short int flush_garbage; // recycled
char user_agent[128];
char referer[256];
char from[256];
char lang_iso[64];
htsrequest_proxy proxy; // proxy
htsrequest_proxy proxy; // proxy
};
// structure pour retour d'une connexion/prise d'en tête
#ifndef HTS_DEF_FWSTRUCT_htsblk
#define HTS_DEF_FWSTRUCT_htsblk
typedef struct htsblk htsblk;
#endif
struct htsblk {
int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945)
short int notmodified; // page ou fichier NON modifié (transféré)
short int is_write; // sortie sur disque (out) ou en mémoire (adr)
short int is_chunk; // mode chunk
short int compressed; // compressé?
short int empty; // vide?
short int keep_alive; // Keep-Alive?
short int keep_alive_trailers; // ..with trailers extension
int keep_alive_t; // KA timeout
int keep_alive_max; // KA number of requests
char* adr; // adresse du bloc de mémoire, NULL=vide
char* headers; // adresse des en têtes si présents
FILE* out; // écriture directe sur disque (si is_write=1)
LLint size; // taille fichier
char msg[80]; // message éventuel si échec ("\0"=non précisé)
char contenttype[64]; // content-type ("text/html" par exemple)
char charset[64]; // charset ("iso-8859-1" par exemple)
char contentencoding[64]; // content-encoding ("gzip" par exemple)
char* location; // on copie dedans éventuellement la véritable 'location'
LLint totalsize; // taille totale à télécharger (-1=inconnue)
short int is_file; // ce n'est pas une socket mais un descripteur de fichier si 1
T_SOC soc; // ID socket
SOCaddr address; // IP address
int address_size; // IP address structure length
FILE* fp; // fichier pour file://
int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945)
short int notmodified; // page ou fichier NON modifié (transféré)
short int is_write; // sortie sur disque (out) ou en mémoire (adr)
short int is_chunk; // mode chunk
short int compressed; // compressé?
short int empty; // vide?
short int keep_alive; // Keep-Alive?
short int keep_alive_trailers; // ..with trailers extension
int keep_alive_t; // KA timeout
int keep_alive_max; // KA number of requests
char *adr; // adresse du bloc de mémoire, NULL=vide
char *headers; // adresse des en têtes si présents
FILE *out; // écriture directe sur disque (si is_write=1)
LLint size; // taille fichier
char msg[80]; // message éventuel si échec ("\0"=non précisé)
char contenttype[64]; // content-type ("text/html" par exemple)
char charset[64]; // charset ("iso-8859-1" par exemple)
char contentencoding[64]; // content-encoding ("gzip" par exemple)
char *location; // on copie dedans éventuellement la véritable 'location'
LLint totalsize; // taille totale à télécharger (-1=inconnue)
short int is_file; // ce n'est pas une socket mais un descripteur de fichier si 1
T_SOC soc; // ID socket
SOCaddr address; // IP address
int address_size; // IP address structure length
FILE *fp; // fichier pour file://
#if HTS_USEOPENSSL
short int ssl; // is this connection a SSL one? (https)
short int ssl; // is this connection a SSL one? (https)
// BIO* ssl_soc; // SSL structure
SSL * ssl_con; // connection structure
SSL *ssl_con; // connection structure
#endif
char lastmodified[64]; // Last-Modified
char etag[64]; // Etag
char cdispo[256]; // Content-Disposition coupé
LLint crange; // Content-Range
LLint crange_start; // Content-Range
LLint crange_end; // Content-Range
int debugid; // debug connection
char lastmodified[64]; // Last-Modified
char etag[64]; // Etag
char cdispo[256]; // Content-Disposition coupé
LLint crange; // Content-Range
LLint crange_start; // Content-Range
LLint crange_end; // Content-Range
int debugid; // debug connection
/* */
htsrequest req; // paramètres pour la requête
/*char digest[32+2]; // digest md5 généré par le moteur ("" si non généré)*/
htsrequest req; // paramètres pour la requête
/*char digest[32+2]; // digest md5 généré par le moteur ("" si non généré) */
};
/* ANCIENNE STURCTURE pour cache 1.0 */
#ifndef HTS_DEF_FWSTRUCT_OLD_t_proxy
#define HTS_DEF_FWSTRUCT_OLD_t_proxy
@@ -185,30 +182,32 @@ struct OLD_t_proxy {
int active;
char name[1024];
int port;
};
};
#ifndef HTS_DEF_FWSTRUCT_OLD_htsblk
#define HTS_DEF_FWSTRUCT_OLD_htsblk
typedef struct OLD_htsblk OLD_htsblk;
#endif
struct OLD_htsblk {
int statuscode; // ANCIENNE STURCTURE - status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945)
int notmodified; // ANCIENNE STURCTURE - page ou fichier NON modifié (transféré)
int is_write; // ANCIENNE STURCTURE - sortie sur disque (out) ou en mémoire (adr)
char* adr; // ANCIENNE STURCTURE - adresse du bloc de mémoire, NULL=vide
FILE* out; // ANCIENNE STURCTURE - écriture directe sur disque (si is_write=1)
int size; // ANCIENNE STURCTURE - taille fichier
char msg[80]; // ANCIENNE STURCTURE - message éventuel si échec ("\0"=non précisé)
char contenttype[64]; // ANCIENNE STURCTURE - content-type ("text/html" par exemple)
char* location; // ANCIENNE STURCTURE - on copie dedans éventuellement la véritable 'location'
int totalsize; // ANCIENNE STURCTURE - taille totale à télécharger (-1=inconnue)
int is_file; // ANCIENNE STURCTURE - ce n'est pas une socket mais un descripteur de fichier si 1
T_SOC soc; // ANCIENNE STURCTURE - ID socket
FILE* fp; // ANCIENNE STURCTURE - fichier pour file://
OLD_t_proxy proxy; // ANCIENNE STURCTURE - proxy
int user_agent_send; // ANCIENNE STURCTURE - user agent (ex: httrack/1.0 [sun])
int statuscode; // ANCIENNE STURCTURE - status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945)
int notmodified; // ANCIENNE STURCTURE - page ou fichier NON modifié (transféré)
int is_write; // ANCIENNE STURCTURE - sortie sur disque (out) ou en mémoire (adr)
char *adr; // ANCIENNE STURCTURE - adresse du bloc de mémoire, NULL=vide
FILE *out; // ANCIENNE STURCTURE - écriture directe sur disque (si is_write=1)
int size; // ANCIENNE STURCTURE - taille fichier
char msg[80]; // ANCIENNE STURCTURE - message éventuel si échec ("\0"=non précisé)
char contenttype[64]; // ANCIENNE STURCTURE - content-type ("text/html" par exemple)
char *location; // ANCIENNE STURCTURE - on copie dedans éventuellement la véritable 'location'
int totalsize; // ANCIENNE STURCTURE - taille totale à télécharger (-1=inconnue)
int is_file; // ANCIENNE STURCTURE - ce n'est pas une socket mais un descripteur de fichier si 1
T_SOC soc; // ANCIENNE STURCTURE - ID socket
FILE *fp; // ANCIENNE STURCTURE - fichier pour file://
OLD_t_proxy proxy; // ANCIENNE STURCTURE - proxy
int user_agent_send; // ANCIENNE STURCTURE - user agent (ex: httrack/1.0 [sun])
char user_agent[64];
int http11; // ANCIENNE STURCTURE - l'en tête doit être signé HTTP/1.1 et non HTTP/1.0
int http11; // ANCIENNE STURCTURE - l'en tête doit être signé HTTP/1.1 et non HTTP/1.0
};
/* fin ANCIENNE STURCTURE pour cache 1.0 */
// cache pour le dns, pour éviter de faire des gethostbyname sans arrêt
@@ -218,24 +217,24 @@ typedef struct t_dnscache t_dnscache;
#endif
struct t_dnscache {
char iadr[1024];
struct t_dnscache* n;
char host_addr[HTS_MAXADDRLEN]; // 4 octets (v4), ou 16 octets (v6)
int host_length; // 4 normalement - ==0 alors en cours de résolution
// ou >16 si sockaddr
// ==-1 alors erreur (host n'éxiste pas)
struct t_dnscache *n;
char host_addr[HTS_MAXADDRLEN]; // 4 octets (v4), ou 16 octets (v6)
int host_length; // 4 normalement - ==0 alors en cours de résolution
// ou >16 si sockaddr
// ==-1 alors erreur (host n'éxiste pas)
};
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
extern htsmutex dns_lock;
// initialize an htsblk structure
void hts_init_htsblk(htsblk* r);
void hts_init_htsblk(htsblk * r);
// fonctions unix/winsock
int hts_read(htsblk* r,char* buff,int size);
int hts_read(htsblk * r, char *buff, int size);
//int HTS_TOTAL_RECV_CHECK(int var);
LLint check_downloadable_bytes(int rate);
@@ -243,207 +242,233 @@ LLint check_downloadable_bytes(int rate);
HTSEXT_API int hts_init(void);
HTSEXT_API int hts_uninit(void);
HTSEXT_API int hts_uninit_module(void);
HTSEXT_API int hts_resetvar(void); /* dummy */
HTSEXT_API int hts_resetvar(void); /* dummy */
HTSEXT_API void hts_debug(int level);
HTSEXT_API httrackp* hts_create_opt(void);
HTSEXT_API void hts_free_opt(httrackp *opt);
HTSEXT_API void set_wrappers(httrackp *opt); /* LEGACY */
HTSEXT_API int plug_wrapper(httrackp *opt, const char *moduleName, const char* argv);
HTSEXT_API httrackp *hts_create_opt(void);
HTSEXT_API void hts_free_opt(httrackp * opt);
HTSEXT_API void set_wrappers(httrackp * opt); /* LEGACY */
HTSEXT_API int plug_wrapper(httrackp * opt, const char *moduleName,
const char *argv);
HTSEXT_API char* hts_strdup(const char* string);
HTSEXT_API void* hts_malloc(size_t size);
HTSEXT_API void* hts_realloc(void* data, size_t size);
HTSEXT_API void hts_free(void* data);
HTSEXT_API char *hts_strdup(const char *string);
HTSEXT_API void *hts_malloc(size_t size);
HTSEXT_API void *hts_realloc(void *data, size_t size);
HTSEXT_API void hts_free(void *data);
#endif
// fonctions principales
T_SOC http_fopen(httrackp *opt,char* adr,char* fil,htsblk* retour);
T_SOC http_xfopen(httrackp *opt,int mode,int treat,int waitconnect,char* xsend,char* adr,char* fil,htsblk* retour);
int http_sendhead(httrackp *opt,t_cookie* cookie,int mode,char* xsend,char* adr,char* fil,char* referer_adr,char* referer_fil,htsblk* retour);
htsblk httpget(httrackp *opt,char* url);
T_SOC http_fopen(httrackp * opt, char *adr, char *fil, htsblk * retour);
T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
char *xsend, char *adr, char *fil, htsblk * retour);
int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, char *xsend,
char *adr, char *fil, char *referer_adr, char *referer_fil,
htsblk * retour);
htsblk httpget(httrackp * opt, char *url);
//int newhttp(char* iadr,char* err=NULL);
T_SOC newhttp(httrackp *opt,const char* iadr,htsblk* retour,int port,int waitconnect);
HTS_INLINE void deletehttp(htsblk* r);
HTS_INLINE int deleteaddr(htsblk* r);
T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
int waitconnect);
HTS_INLINE void deletehttp(htsblk * r);
HTS_INLINE int deleteaddr(htsblk * r);
HTS_INLINE void deletesoc(T_SOC soc);
HTS_INLINE void deletesoc_r(htsblk* r);
htsblk http_location(httrackp *opt,char* adr,char* fil,char* loc);
htsblk http_test(httrackp *opt,char* adr,char* fil,char* loc);
int check_readinput(htsblk* r);
HTS_INLINE void deletesoc_r(htsblk * r);
htsblk http_location(httrackp * opt, char *adr, char *fil, char *loc);
htsblk http_test(httrackp * opt, char *adr, char *fil, char *loc);
int check_readinput(htsblk * r);
int check_readinput_t(T_SOC soc, int timeout);
void http_fread(T_SOC soc,htsblk* retour);
LLint http_fread1(htsblk* r);
void treathead(t_cookie* cookie,char* adr,char* fil,htsblk* retour,char* rcvd);
void treatfirstline(htsblk* retour,char* rcvd);
void http_fread(T_SOC soc, htsblk * retour);
LLint http_fread1(htsblk * r);
void treathead(t_cookie * cookie, char *adr, char *fil, htsblk * retour,
char *rcvd);
void treatfirstline(htsblk * retour, char *rcvd);
#ifndef HTTRACK_DEFLIB
HTSEXT_API void infostatuscode(char* msg,int statuscode);
HTSEXT_API void infostatuscode(char *msg, int statuscode);
#endif
// sous-fonctions
htsblk xhttpget(httrackp *opt,char* adr,char* fil);
htsblk http_gethead(httrackp *opt,char* adr,char* fil);
LLint http_xfread1(htsblk* r,int bufl);
HTS_INLINE t_hostent* hts_gethostbyname(httrackp *opt,const char* iadr, void* v_buffer);
htsblk xhttpget(httrackp * opt, char *adr, char *fil);
htsblk http_gethead(httrackp * opt, char *adr, char *fil);
LLint http_xfread1(htsblk * r, int bufl);
HTS_INLINE t_hostent *hts_gethostbyname(httrackp * opt, const char *iadr,
void *v_buffer);
#ifndef HTTRACK_DEFLIB
HTSEXT_API t_hostent* vxgethostbyname(char* hostname, void* v_buffer);
HTSEXT_API t_hostent *vxgethostbyname(char *hostname, void *v_buffer);
#endif
t_hostent* _hts_ghbn(t_dnscache* cache,const char* iadr,t_hostent* retour);
t_hostent *_hts_ghbn(t_dnscache * cache, const char *iadr, t_hostent * retour);
int ftp_available(void);
#if HTS_DNSCACHE
void hts_cache_free(t_dnscache* cache);
int hts_dnstest(httrackp *opt, const char* _iadr);
t_dnscache* _hts_cache(httrackp *opt);
void hts_cache_free(t_dnscache * cache);
int hts_dnstest(httrackp * opt, const char *_iadr);
t_dnscache *_hts_cache(httrackp * opt);
#endif
// outils divers
HTS_INLINE TStamp time_local(void);
#ifndef HTTRACK_DEFLIB
HTSEXT_API HTS_INLINE TStamp mtime_local(void);
#endif
void sec2str(char *s,TStamp t);
void sec2str(char *s, TStamp t);
#ifndef HTTRACK_DEFLIB
HTSEXT_API void qsec2str(char *st,TStamp t);
HTSEXT_API void qsec2str(char *st, TStamp t);
#endif
void time_gmt_rfc822(char* s);
void time_local_rfc822(char* s);
struct tm* convert_time_rfc822(struct tm* buffer, const char* s);
int set_filetime(const char* file,struct tm* tm_time);
int set_filetime_rfc822(const char* file,const char* date);
int get_filetime_rfc822(const char* file,char* date);
HTS_INLINE void time_rfc822(char* s,struct tm * A);
HTS_INLINE void time_rfc822_local(char* s,struct tm * A);
void time_gmt_rfc822(char *s);
void time_local_rfc822(char *s);
struct tm *convert_time_rfc822(struct tm *buffer, const char *s);
int set_filetime(const char *file, struct tm *tm_time);
int set_filetime_rfc822(const char *file, const char *date);
int get_filetime_rfc822(const char *file, char *date);
HTS_INLINE void time_rfc822(char *s, struct tm *A);
HTS_INLINE void time_rfc822_local(char *s, struct tm *A);
#ifndef HTTRACK_DEFLIB
HTSEXT_API char* int2char(strc_int2bytes2* strc, int n);
HTSEXT_API char* int2bytes(strc_int2bytes2* strc, LLint n);
HTSEXT_API char* int2bytessec(strc_int2bytes2* strc, long int n);
HTSEXT_API char** int2bytes2(strc_int2bytes2* strc, LLint n);
HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n);
HTSEXT_API char *int2bytes(strc_int2bytes2 * strc, LLint n);
HTSEXT_API char *int2bytessec(strc_int2bytes2 * strc, long int n);
HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n);
#endif
HTS_INLINE int sendc(htsblk* r, const char* s);
int finput(int fd,char* s,int max);
int binput(char* buff,char* s,int max);
int linput(FILE* fp,char* s,int max);
int linputsoc(T_SOC soc, char* s, int max);
int linputsoc_t(T_SOC soc, char* s, int max, int timeout);
int linput_trim(FILE* fp,char* s,int max);
int linput_cpp(FILE* fp,char* s,int max);
void rawlinput(FILE* fp,char* s,int max);
char* strstrcase(char *s,char *o);
int ident_url_absolute(const char* url,char* adr,char* fil);
void fil_simplifie(char* f);
int is_unicode_utf8(const char* buffer, size_t size);
void map_characters(unsigned char* buffer, unsigned int size, unsigned int* map);
int ishtml(httrackp *opt,const char* urlfil);
int ishtml_ext(const char* a);
HTS_INLINE int sendc(htsblk * r, const char *s);
int finput(int fd, char *s, int max);
int binput(char *buff, char *s, int max);
int linput(FILE * fp, char *s, int max);
int linputsoc(T_SOC soc, char *s, int max);
int linputsoc_t(T_SOC soc, char *s, int max, int timeout);
int linput_trim(FILE * fp, char *s, int max);
int linput_cpp(FILE * fp, char *s, int max);
void rawlinput(FILE * fp, char *s, int max);
char *strstrcase(char *s, char *o);
int ident_url_absolute(const char *url, char *adr, char *fil);
void fil_simplifie(char *f);
int is_unicode_utf8(const char *buffer, size_t size);
void map_characters(unsigned char *buffer, unsigned int size,
unsigned int *map);
int ishtml(httrackp * opt, const char *urlfil);
int ishtml_ext(const char *a);
int ishttperror(int err);
void guess_httptype(httrackp *opt,char *s,const char *fil);
void guess_httptype(httrackp * opt, char *s, const char *fil);
#ifndef HTTRACK_DEFLIB
HTSEXT_API void get_httptype(httrackp *opt,char *s,const char *fil,int flag);
HTSEXT_API void get_httptype(httrackp * opt, char *s, const char *fil,
int flag);
#endif
int get_userhttptype(httrackp *opt,char *s,const char *fil);
void give_mimext(char *s,const char *st);
int get_userhttptype(httrackp * opt, char *s, const char *fil);
void give_mimext(char *s, const char *st);
#ifndef HTTRACK_DEFLIB
HTSEXT_API int is_knowntype(httrackp *opt,const char *fil);
HTSEXT_API int is_userknowntype(httrackp *opt,const char *fil);
HTSEXT_API int is_knowntype(httrackp * opt, const char *fil);
HTSEXT_API int is_userknowntype(httrackp * opt, const char *fil);
HTSEXT_API int is_dyntype(const char *fil);
HTSEXT_API char* get_ext(char *catbuff, const char *fil);
HTSEXT_API char *get_ext(char *catbuff, const char *fil);
#endif
int may_unknown(httrackp *opt,const char* st);
int may_bogus_multiple(httrackp *opt, const char* mime, const char *filename);
int may_unknown2(httrackp *opt,const char* mime, const char *filename);
int may_unknown(httrackp * opt, const char *st);
int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename);
int may_unknown2(httrackp * opt, const char *mime, const char *filename);
#ifndef HTTRACK_DEFLIB
HTSEXT_API char* jump_identification(const char*);
HTSEXT_API char* jump_normalized(const char*);
HTSEXT_API char* jump_toport(const char*);
HTSEXT_API char* fil_normalized(const char* source, char* dest);
HTSEXT_API char* adr_normalized(const char* source, char* dest);
HTSEXT_API char *jump_identification(const char *);
HTSEXT_API char *jump_normalized(const char *);
HTSEXT_API char *jump_toport(const char *);
HTSEXT_API char *fil_normalized(const char *source, char *dest);
HTSEXT_API char *adr_normalized(const char *source, char *dest);
#endif
char* strrchr_limit(const char* s, char c, const char* limit);
char* strstr_limit(const char* s, const char* sub, const char* limit);
HTS_INLINE char* jump_protocol(const char* source);
void code64(unsigned char* a,int size_a,unsigned char* b,int crlf);
char *strrchr_limit(const char *s, char c, const char *limit);
char *strstr_limit(const char *s, const char *sub, const char *limit);
HTS_INLINE char *jump_protocol(const char *source);
void code64(unsigned char *a, int size_a, unsigned char *b, int crlf);
#ifndef HTTRACK_DEFLIB
HTSEXT_API void unescape_amp(char* s);
HTSEXT_API void escape_spc_url(char* s);
HTSEXT_API void escape_in_url(char* s);
HTSEXT_API void escape_uri(char* s);
HTSEXT_API void escape_uri_utf(char* s);
HTSEXT_API void escape_check_url(char* s);
HTSEXT_API char* escape_check_url_addr(char *catbuff, const char* s);
HTSEXT_API void x_escape_http(char* s,int mode);
HTSEXT_API void x_escape_html(char* s);
HTSEXT_API void escape_remove_control(char* s);
HTSEXT_API void escape_for_html_print(char* s, char* d);
HTSEXT_API void escape_for_html_print_full(char* s, char* d);
HTSEXT_API void unescape_amp(char *s);
HTSEXT_API void escape_spc_url(char *s);
HTSEXT_API void escape_in_url(char *s);
HTSEXT_API void escape_uri(char *s);
HTSEXT_API void escape_uri_utf(char *s);
HTSEXT_API void escape_check_url(char *s);
HTSEXT_API char *escape_check_url_addr(char *catbuff, const char *s);
HTSEXT_API void x_escape_http(char *s, int mode);
HTSEXT_API void x_escape_html(char *s);
HTSEXT_API void escape_remove_control(char *s);
HTSEXT_API void escape_for_html_print(char *s, char *d);
HTSEXT_API void escape_for_html_print_full(char *s, char *d);
#endif
#ifndef HTTRACK_DEFLIB
HTSEXT_API char* unescape_http(char *catbuff, const char* s);
HTSEXT_API char* unescape_http_unharm(char *catbuff, const char* s, int no_high);
HTSEXT_API char* antislash_unescaped(char *catbuff, const char* s);
HTSEXT_API char* concat(char *catbuff,const char* a,const char* b);
HTSEXT_API char* fconcat(char *catbuff, const char* a, const char* b);
HTSEXT_API char* fconv(char *catbuff, const char* a);
HTSEXT_API char *unescape_http(char *catbuff, const char *s);
HTSEXT_API char *unescape_http_unharm(char *catbuff, const char *s,
int no_high);
HTSEXT_API char *antislash_unescaped(char *catbuff, const char *s);
HTSEXT_API char *concat(char *catbuff, const char *a, const char *b);
HTSEXT_API char *fconcat(char *catbuff, const char *a, const char *b);
HTSEXT_API char *fconv(char *catbuff, const char *a);
#endif
#define copychar(catbuff,a) concat(catbuff,(a),NULL)
char* fslash(char *catbuff, const char* a);
char *fslash(char *catbuff, const char *a);
#ifndef HTTRACK_DEFLIB
HTSEXT_API int hts_log(httrackp *opt, const char* prefix, const char *msg);
HTSEXT_API void hts_log_print(httrackp *opt, int type, const char *format, ...) HTS_PRINTF_FUN(3, 4);
HTSEXT_API int hts_log(httrackp * opt, const char *prefix, const char *msg);
HTSEXT_API void hts_log_print(httrackp * opt, int type, const char *format,
...) HTS_PRINTF_FUN(3, 4);
#endif
char* convtolower(char *catbuff, const char* a);
void hts_lowcase(char* s);
void hts_replace(char *s,char from,char to);
int multipleStringMatch(const char *s, const char *match);
char *convtolower(char *catbuff, const char *a);
void hts_lowcase(char *s);
void hts_replace(char *s, char from, char to);
int multipleStringMatch(const char *s, const char *match);
void fprintfio(FILE* fp,char* buff,char* prefix);
void fprintfio(FILE * fp, char *buff, char *prefix);
#ifdef _WIN32
#else
int sig_ignore_flag( int setflag ); // flag ignore
int sig_ignore_flag(int setflag); // flag ignore
#endif
void cut_path(char* fullpath,char* path,char* pname);
int fexist(const char* s);
int fexist_utf8(const char* s);
void cut_path(char *fullpath, char *path, char *pname);
int fexist(const char *s);
int fexist_utf8(const char *s);
/*LLint fsize(const char* s); */
off_t fpsize(FILE* fp);
off_t fsize(const char* s);
off_t fsize_utf8(const char* s);
off_t fpsize(FILE * fp);
off_t fsize(const char *s);
off_t fsize_utf8(const char *s);
/* root dir */
#ifndef HTTRACK_DEFLIB
HTSEXT_API char* hts_rootdir(char* file);
HTSEXT_API char *hts_rootdir(char *file);
#endif
// Threads
typedef void* ( *beginthread_type )( void * );
typedef void *(*beginthread_type) (void *);
/*unsigned long _beginthread( beginthread_type start_address, unsigned stack_size, void *arglist );*/
/* variables globales */
extern HTSEXT_API hts_stat_struct HTS_STAT;
extern int _DEBUG_HEAD;
extern FILE* ioinfo;
extern HTSEXT_API hts_stat_struct HTS_STAT;
extern int _DEBUG_HEAD;
extern FILE *ioinfo;
/* constantes */
extern const char* hts_mime_keep[];
extern const char* hts_mime[][2];
extern const char* hts_main_mime[];
extern const char* hts_detect[];
extern const char* hts_detectbeg[];
extern const char* hts_nodetect[];
extern const char* hts_detectURL[];
extern const char* hts_detectandleave[];
extern const char* hts_detect_js[];
extern const char *hts_mime_keep[];
extern const char *hts_mime[][2];
extern const char *hts_main_mime[];
extern const char *hts_detect[];
extern const char *hts_detectbeg[];
extern const char *hts_nodetect[];
extern const char *hts_detectURL[];
extern const char *hts_detectandleave[];
extern const char *hts_detect_js[];
// htsmodule.c definitions
extern void* openFunctionLib(const char* file_);
extern void* getFunctionPtr(void* handle, const char* fncname);
extern void closeFunctionLib(void* handle);
extern void *openFunctionLib(const char *file_);
extern void *getFunctionPtr(void *handle, const char *fncname);
extern void closeFunctionLib(void *handle);
extern void clearCallbacks(htscallbacks* chain);
extern size_t hts_get_callback_offs(const char *name);
int hts_set_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name, void *function);
void *hts_get_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name);
extern void clearCallbacks(htscallbacks * chain);
extern size_t hts_get_callback_offs(const char *name);
int hts_set_callback(t_hts_htmlcheck_callbacks * callbacks,
const char *name, void *function);
void *hts_get_callback(t_hts_htmlcheck_callbacks * callbacks,
const char *name);
#define CBSTRUCT(OPT) ((t_hts_htmlcheck_callbacks*) ((OPT)->callbacks_fun))
#define GET_USERCALLBACK(OPT, NAME) ( CBSTRUCT(OPT)-> NAME .fun )
@@ -488,34 +513,42 @@ void *hts_get_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name);
#ifndef HTS_DEF_FILEAPI
#ifdef _WIN32
#define FOPEN hts_fopen_utf8
HTSEXT_API FILE* hts_fopen_utf8(const char *path, const char *mode);
HTSEXT_API FILE *hts_fopen_utf8(const char *path, const char *mode);
#define STAT hts_stat_utf8
typedef struct _stat STRUCT_STAT;
HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT *buf);
typedef struct _stat STRUCT_STAT;
HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT * buf);
#define UNLINK hts_unlink_utf8
HTSEXT_API int hts_unlink_utf8(const char *pathname);
HTSEXT_API int hts_unlink_utf8(const char *pathname);
#define RENAME hts_rename_utf8
HTSEXT_API int hts_rename_utf8(const char *oldpath, const char *newpath);
HTSEXT_API int hts_rename_utf8(const char *oldpath, const char *newpath);
#define MKDIR(F) hts_mkdir_utf8(F)
HTSEXT_API int hts_mkdir_utf8(const char *pathname);
HTSEXT_API int hts_mkdir_utf8(const char *pathname);
#define UTIME(A,B) hts_utime_utf8(A,B)
typedef struct _utimbuf STRUCT_UTIMBUF;
HTSEXT_API int hts_utime_utf8(const char *filename, const STRUCT_UTIMBUF *times);
typedef struct _utimbuf STRUCT_UTIMBUF;
HTSEXT_API int hts_utime_utf8(const char *filename,
const STRUCT_UTIMBUF * times);
#else
/* The underlying filesystem charset is supposed to be UTF-8 */
#define FOPEN fopen
#define STAT stat
typedef struct stat STRUCT_STAT;
typedef struct stat STRUCT_STAT;
#define UNLINK unlink
#define RENAME rename
#define MKDIR(F) mkdir(F, HTS_ACCESS_FOLDER)
typedef struct utimbuf STRUCT_UTIMBUF;
typedef struct utimbuf STRUCT_UTIMBUF;
#define UTIME(A,B) utime(A,B)
#endif
#define HTS_DEF_FILEAPI
#endif
#endif // internals
#endif // internals
#undef PATH_SEPARATOR
#ifdef _WIN32
@@ -547,17 +580,23 @@ typedef struct utimbuf STRUCT_UTIMBUF;
// compare le début de f avec s et retourne la position de la fin
// 'A=a' (case insensitive)
HTS_STATIC int strfield(const char* f,const char* s) {
int r=0;
while (streql(*f,*s) && ((*f)!=0) && ((*s)!=0)) { f++; s++; r++; }
if (*s==0)
HTS_STATIC int strfield(const char *f, const char *s) {
int r = 0;
while(streql(*f, *s) && ((*f) != 0) && ((*s) != 0)) {
f++;
s++;
r++;
}
if (*s == 0)
return r;
else
return 0;
}
HTS_STATIC int strcmpnocase(char* a,char* b) {
}
HTS_STATIC int strcmpnocase(char *a, char *b) {
while(*a) {
int cmp = hichar(*a) - hichar(*b);
if (cmp != 0)
return cmp;
a++;
@@ -609,39 +648,45 @@ HTS_STATIC int strcmpnocase(char* a,char* b) {
#ifdef HTS_INTERNAL_BYTECODE
// check if (mime, file) is hypertext
HTS_STATIC int is_hypertext_mime(httrackp *opt,const char* mime, const char* file) {
HTS_STATIC int is_hypertext_mime(httrackp * opt, const char *mime,
const char *file) {
if (is_hypertext_mime__(mime))
return 1;
if (may_unknown(opt,mime)) {
if (may_unknown(opt, mime)) {
char guessed[256];
guessed[0] = '\0';
guess_httptype(opt,guessed, file);
guess_httptype(opt, guessed, file);
return is_hypertext_mime__(guessed);
}
return 0;
}
// check if (mime, file) might be "false" hypertext
HTS_STATIC int may_be_hypertext_mime(httrackp *opt,const char* mime, const char* file) {
HTS_STATIC int may_be_hypertext_mime(httrackp * opt, const char *mime,
const char *file) {
if (may_be_hypertext_mime__(mime))
return 1;
if (file != NULL && file[0] != '\0' && may_unknown(opt,mime)) {
if (file != NULL && file[0] != '\0' && may_unknown(opt, mime)) {
char guessed[256];
guessed[0] = '\0';
guess_httptype(opt,guessed, file);
guess_httptype(opt, guessed, file);
return may_be_hypertext_mime__(guessed);
}
return 0;
}
// compare (mime, file) with reference
HTS_STATIC int compare_mime(httrackp *opt,const char* mime, const char* file, const char* reference) {
HTS_STATIC int compare_mime(httrackp * opt, const char *mime, const char *file,
const char *reference) {
if (is_hypertext_mime__(mime) || may_be_hypertext_mime__(mime))
return strfield2(mime, reference);
if (file != NULL && file[0] != '\0' && may_unknown(opt,mime)) {
if (file != NULL && file[0] != '\0' && may_unknown(opt, mime)) {
char guessed[256];
guessed[0] = '\0';
guess_httptype(opt,guessed, file);
guess_httptype(opt, guessed, file);
return strfield2(guessed, reference);
}
return 0;
@@ -650,26 +695,26 @@ HTS_STATIC int compare_mime(httrackp *opt,const char* mime, const char* file, co
#endif
#ifdef _WIN32_WCE_XXC
extern char cwd[MAX_PATH+1];
extern char cwd[MAX_PATH + 1];
HTS_STATIC char *getcwd_ce(char *buffer, int maxlen) {
TCHAR fileUnc[MAX_PATH+1];
char* plast;
if(cwd[0] == 0)
{
GetModuleFileName(NULL, fileUnc, MAX_PATH);
WideCharToMultiByte(CP_ACP, 0, fileUnc, -1, cwd, MAX_PATH, NULL, NULL);
plast = strrchr(cwd, '\\');
if(plast)
*plast = 0;
/* Special trick to keep start menu clean... */
if(_stricmp(cwd, "\\windows\\start menu") == 0)
strcpy(cwd, "\\Apps");
}
if(buffer)
strncpy(buffer, cwd, maxlen);
return cwd;
TCHAR fileUnc[MAX_PATH + 1];
char *plast;
if (cwd[0] == 0) {
GetModuleFileName(NULL, fileUnc, MAX_PATH);
WideCharToMultiByte(CP_ACP, 0, fileUnc, -1, cwd, MAX_PATH, NULL, NULL);
plast = strrchr(cwd, '\\');
if (plast)
*plast = 0;
/* Special trick to keep start menu clean... */
if (_stricmp(cwd, "\\windows\\start menu") == 0)
strcpy(cwd, "\\Apps");
}
if (buffer)
strncpy(buffer, cwd, maxlen);
return cwd;
}
#undef getcwd
#define getcwd getcwd_ce
#endif
@@ -678,11 +723,11 @@ HTS_STATIC char *getcwd_ce(char *buffer, int maxlen) {
#ifdef _WIN32
#define HTS_DIRENT_SIZE 256
struct dirent {
ino_t d_ino; /* ignored */
off_t d_off; /* ignored */
unsigned short d_reclen; /* ignored */
unsigned char d_type; /* ignored */
char d_name[HTS_DIRENT_SIZE]; /* filename */
ino_t d_ino; /* ignored */
off_t d_off; /* ignored */
unsigned short d_reclen; /* ignored */
unsigned char d_type; /* ignored */
char d_name[HTS_DIRENT_SIZE]; /* filename */
};
typedef struct DIR DIR;
struct DIR {
@@ -691,8 +736,8 @@ struct DIR {
char *name;
};
DIR *opendir(const char *name);
struct dirent *readdir(DIR *dir);
int closedir(DIR *dir);
struct dirent *readdir(DIR * dir);
int closedir(DIR * dir);
#endif
#endif

27
src/htsmd5.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsmd5.c subroutines: */
/* generate a md5 hash */
@@ -48,34 +45,36 @@ Please visit our Website: http://www.httrack.com
#include "htsmd5.h"
#include "md5.h"
int domd5mem(const char * buf, size_t len, char * digest, int asAscii) {
int domd5mem(const char *buf, size_t len, char *digest, int asAscii) {
int endian = 1;
unsigned char bindigest[16];
MD5_CTX ctx;
MD5Init(&ctx, * ( (char*) &endian));
MD5Update(&ctx, (const unsigned char*) buf, (unsigned int) len);
MD5Init(&ctx, *((char *) &endian));
MD5Update(&ctx, (const unsigned char *) buf, (unsigned int) len);
MD5Final(bindigest, &ctx);
if (!asAscii) {
memcpy(digest, bindigest, 16);
} else {
sprintf(digest, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
"%02x%02x%02x%02x%02x",
bindigest[0], bindigest[1], bindigest[2], bindigest[3],
bindigest[4], bindigest[5], bindigest[6], bindigest[7],
bindigest[8], bindigest[9], bindigest[10], bindigest[11],
bindigest[12], bindigest[13], bindigest[14], bindigest[15]);
sprintf(digest,
"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
"%02x%02x%02x%02x%02x", bindigest[0], bindigest[1], bindigest[2],
bindigest[3], bindigest[4], bindigest[5], bindigest[6],
bindigest[7], bindigest[8], bindigest[9], bindigest[10],
bindigest[11], bindigest[12], bindigest[13], bindigest[14],
bindigest[15]);
}
return 0;
}
unsigned long int md5sum32(const char* buff) {
unsigned long int md5sum32(const char *buff) {
union {
char md5digest[16];
unsigned long int hash;
} u;
domd5mem(buff, strlen(buff), u.md5digest, 0);
return u.hash;
}

12
src/htsmd5.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsmd5.c subroutines: */
/* generate a md5 hash */
@@ -40,15 +37,12 @@ Please visit our Website: http://www.httrack.com
/* ------------------------------------------------------------ */
#ifndef HTSMD5_DEFH
#define HTSMD5_DEFH
#define HTSMD5_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
int domd5mem(const char * buf, size_t len, char * digest, int asAscii);
unsigned long int md5sum32(const char* buff);
int domd5mem(const char *buf, size_t len, char *digest, int asAscii);
unsigned long int md5sum32(const char *buff);
#endif
#endif

299
src/htsmms.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: basic mms protocol manager */
/* Author: Xavier Roche */
@@ -54,179 +51,185 @@ Please visit our Website: http://www.httrack.com
#define FTP_STATUS_READY 1001
static int run_launch_mms(MMSDownloadStruct* back);
static void back_launch_mms( void* pP ) {
MMSDownloadStruct *pStruct = (MMSDownloadStruct*)pP;
static int run_launch_mms(MMSDownloadStruct * back);
static void back_launch_mms(void *pP) {
MMSDownloadStruct *pStruct = (MMSDownloadStruct *) pP;
if (pStruct == NULL)
return ;
return;
/* Initialize */
hts_init();
/* Initialize */
hts_init();
/* Run */
run_launch_mms(pStruct);
/* Run */
run_launch_mms(pStruct);
/* Set as ready */
{
lien_back* back = pStruct->pBack;
back->status=FTP_STATUS_READY;
}
/* Set as ready */
{
lien_back *back = pStruct->pBack;
/* Delete structure */
free(pP);
back->status = FTP_STATUS_READY;
}
/* Uninitialize */
/* Delete structure */
free(pP);
/* Uninitialize */
hts_uninit();
return ;
return;
}
/* download cancelled */
static int stop_mms(lien_back* back) {
if (back->stop_ftp) {
strcpybuff(back->r.msg, "Cancelled by User");
back->r.statuscode = STATUSCODE_INVALID;
return 1;
}
return 0;
static int stop_mms(lien_back * back) {
if (back->stop_ftp) {
strcpybuff(back->r.msg, "Cancelled by User");
back->r.statuscode = STATUSCODE_INVALID;
return 1;
}
return 0;
}
/* Background launch */
void launch_mms(const MMSDownloadStruct* pStruct) {
MMSDownloadStruct *pCopy = calloc(sizeof(MMSDownloadStruct), 1);
memcpy(pCopy, pStruct, sizeof(*pCopy));
hts_newthread(back_launch_mms, (void*) pCopy);
void launch_mms(const MMSDownloadStruct * pStruct) {
MMSDownloadStruct *pCopy = calloc(sizeof(MMSDownloadStruct), 1);
memcpy(pCopy, pStruct, sizeof(*pCopy));
hts_newthread(back_launch_mms, (void *) pCopy);
}
/* Code mainly written by Nicolas BENOIT */
static int run_launch_mms(MMSDownloadStruct* pStruct) {
lien_back* back = pStruct->pBack;
httrackp* opt = pStruct->pOpt;
/* */
char url[HTS_URLMAXSIZE*2];
char catbuff[CATBUFF_SIZE];
char catbuff2[CATBUFF_SIZE];
static int run_launch_mms(MMSDownloadStruct * pStruct) {
lien_back *back = pStruct->pBack;
httrackp *opt = pStruct->pOpt;
/* */
char url[HTS_URLMAXSIZE * 2];
char catbuff[CATBUFF_SIZE];
char catbuff2[CATBUFF_SIZE];
MMS *mms;
FILE *f;
ssize_t len_written;
uint64_t total_len_written;
int delay = opt->mms_maxtime;
time_t end = time(NULL) + delay;
short checkPending = 0;
ssize_t existingSize = fsize(back->url_sav);
uint64_t total_len_written;
int delay = opt->mms_maxtime;
time_t end = time(NULL) + delay;
short checkPending = 0;
ssize_t existingSize = fsize(back->url_sav);
// effacer
strcpybuff(back->r.msg,"");
back->status=STATUS_FTP_TRANSFER;
back->r.statuscode=HTTP_OK;
back->r.size=0;
// effacer
strcpybuff(back->r.msg, "");
back->status = STATUS_FTP_TRANSFER;
back->r.statuscode = HTTP_OK;
back->r.size = 0;
/* Create file */
if (existingSize > 0) {
/* back->r.out = fileappend(back->url_sav);
*/
(void) unlink(fconcat(catbuff,back->url_sav, ".old"));
if (rename(fconcat(catbuff,back->url_sav, ""), fconcat(catbuff2,back->url_sav, ".old")) == 0) {
checkPending = 1;
}
back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav);
} else {
back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav);
}
if ((f = back->r.out) != NULL) {
// create mms resource
strcpybuff(url, back->url_adr); /* mms:// */
strcatbuff(url, back->url_fil);
if ( ( mms = mms_create( url, f, NULL, 0, 1 ) ) != NULL ) {
if ( mms_connect ( mms ) == 0 ) {
if ( mms_handshake ( mms ) == 0 ) {
if ( ( len_written = mms_write_stream_header ( mms ) ) != -1 ) {
total_len_written = len_written;
HTS_STAT.HTS_TOTAL_RECV += len_written;
/* Create file */
if (existingSize > 0) {
/* back->r.out = fileappend(back->url_sav);
*/
(void) unlink(fconcat(catbuff, back->url_sav, ".old"));
if (rename
(fconcat(catbuff, back->url_sav, ""),
fconcat(catbuff2, back->url_sav, ".old")) == 0) {
checkPending = 1;
}
back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav);
} else {
back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav);
}
if ((f = back->r.out) != NULL) {
// create mms resource
strcpybuff(url, back->url_adr); /* mms:// */
strcatbuff(url, back->url_fil);
if ((mms = mms_create(url, f, NULL, 0, 1)) != NULL) {
if (mms_connect(mms) == 0) {
if (mms_handshake(mms) == 0) {
if ((len_written = mms_write_stream_header(mms)) != -1) {
total_len_written = len_written;
HTS_STAT.HTS_TOTAL_RECV += len_written;
/* not modified */
if (checkPending) {
if (mms->is_live != MMS_LIVE
&& mms->expected_file_size == existingSize + 50 /* Why 50 additional bytes declared ?? */
) // abort download
{
fclose(back->r.out);
f = back->r.out = NULL;
if (unlink(fconcat(catbuff, back->url_sav, "")) == 0
&& rename(fconcat(catbuff, back->url_sav, ".old"), fconcat(catbuff2, back->url_sav, "")) == 0)
{
back->r.notmodified = 1;
back->r.statuscode = HTTP_OK;
strcpybuff(back->r.msg, "Not modified");
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Unable to rename previous file (not updated)");
}
} else {
(void) unlink(fconcat(catbuff, back->url_sav, ".old"));
}
}
/* not modified */
if (checkPending) {
if (mms->is_live != MMS_LIVE && mms->expected_file_size == existingSize + 50 /* Why 50 additional bytes declared ?? */
) // abort download
{
fclose(back->r.out);
f = back->r.out = NULL;
if (unlink(fconcat(catbuff, back->url_sav, "")) == 0
&& rename(fconcat(catbuff, back->url_sav, ".old"),
fconcat(catbuff2, back->url_sav, "")) == 0) {
back->r.notmodified = 1;
back->r.statuscode = HTTP_OK;
strcpybuff(back->r.msg, "Not modified");
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg,
"Unable to rename previous file (not updated)");
}
} else {
(void) unlink(fconcat(catbuff, back->url_sav, ".old"));
}
}
/* begin rip */
if ( f != NULL && mms_begin_rip ( mms ) == 0 ) {
if ( mms->is_live != MMS_LIVE ) {
back->r.totalsize = mms->expected_file_size;
back->r.totalsize -= 50; /* Why 50 additional bytes declared ?? */
} else
back->r.totalsize = -1;
/* begin rip */
if (f != NULL && mms_begin_rip(mms) == 0) {
if (mms->is_live != MMS_LIVE) {
back->r.totalsize = mms->expected_file_size;
back->r.totalsize -= 50; /* Why 50 additional bytes declared ?? */
} else
back->r.totalsize = -1;
/* Start download */
while ( !stop_mms(back) ) {
len_written = mms_write_stream_data ( mms );
if ( len_written == 0 ) {
break;
} else if ( len_written == -1 ) {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Unable to write stream data");
break;
}
/* Start download */
while(!stop_mms(back)) {
len_written = mms_write_stream_data(mms);
if (len_written == 0) {
break;
} else if (len_written == -1) {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Unable to write stream data");
break;
}
total_len_written += len_written;
back->r.size = total_len_written;
HTS_STAT.HTS_TOTAL_RECV += len_written;
total_len_written += len_written;
back->r.size = total_len_written;
HTS_STAT.HTS_TOTAL_RECV += len_written;
fflush ( f );
fflush(f);
if ( delay != 0 && end <= time(NULL) ) {
delay = -1;
back->r.statuscode = HTTP_OK;
strcpybuff(back->r.msg, "Download interrupted");
break;
}
} // while
if (delay != 0 && end <= time(NULL)) {
delay = -1;
back->r.statuscode = HTTP_OK;
strcpybuff(back->r.msg, "Download interrupted");
break;
}
} // while
back->r.statuscode = HTTP_OK; /* Finished */
} else if (f != NULL) {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not begin ripping");
}
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not write stream header");
}
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not handshake");
}
mms_disconnect ( mms );
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not connect");
}
mms_destroy ( mms );
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not create mms resource");
}
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Unable to open local output file");
}
return 0;
back->r.statuscode = HTTP_OK; /* Finished */
} else if (f != NULL) {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not begin ripping");
}
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not write stream header");
}
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not handshake");
}
mms_disconnect(mms);
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not connect");
}
mms_destroy(mms);
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Can not create mms resource");
}
} else {
back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR;
strcpybuff(back->r.msg, "Unable to open local output file");
}
return 0;
}
#endif

12
src/htsmms.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: basic mms protocol manager .h */
/* Author: Xavier Roche */
@@ -38,9 +35,8 @@ Please visit our Website: http://www.httrack.com
/* Thanks to all of them! */
/* ------------------------------------------------------------ */
#ifndef HTSMMS_DEFH
#define HTSMMS_DEFH
#define HTSMMS_DEFH
#if HTS_USEMMS
@@ -59,11 +55,11 @@ typedef struct httrackp httrackp;
typedef struct MMSDownloadStruct MMSDownloadStruct;
#endif
struct MMSDownloadStruct {
lien_back *pBack;
httrackp *pOpt;
lien_back *pBack;
httrackp *pOpt;
};
void launch_mms(const MMSDownloadStruct* pStruct);
void launch_mms(const MMSDownloadStruct * pStruct);
#endif
#endif

197
src/htsmodules.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsmodules.c subroutines: */
/* external modules (parsers) */
@@ -44,7 +41,7 @@ Please visit our Website: http://www.httrack.com
#include "htsbasenet.h"
#include "htslib.h"
extern int fspc(httrackp *opt,FILE* fp,const char* type);
extern int fspc(httrackp * opt, FILE * fp, const char *type);
#ifndef _WIN32
#if HTS_DLOPEN
@@ -60,6 +57,7 @@ extern int fspc(httrackp *opt,FILE* fp,const char* type);
/* >>> Put all modules variables here */
int gz_is_available = 0;
#if 0
t_gzopen gzopen = NULL;
t_gzread gzread = NULL;
@@ -88,18 +86,22 @@ t_SSL_load_error_strings SSL_load_error_strings = NULL;
int V6_is_available = HTS_INET6;
static char WHAT_is_available[64]="";
static char WHAT_is_available[64] = "";
/* <<< */
HTSEXT_API const char* hts_get_version_info(httrackp *opt) {
HTSEXT_API const char *hts_get_version_info(httrackp * opt) {
size_t size;
int i;
strcpy(opt->state.HTbuff, WHAT_is_available);
size = strlen(opt->state.HTbuff);
for(i = 0 ; i < opt->libHandles.count ; i++) {
for(i = 0; i < opt->libHandles.count; i++) {
const char *name = opt->libHandles.handles[i].moduleName;
if (name != NULL) {
size_t nsize = strlen(name) + sizeof("+");
size += nsize;
if (size + 1 >= sizeof(opt->state.HTbuff))
break;
@@ -113,24 +115,30 @@ HTSEXT_API const char* hts_get_version_info(httrackp *opt) {
/* memory checks */
HTSEXT_API htsErrorCallback htsCallbackErr = NULL;
HTSEXT_API int htsMemoryFastXfr = 1; /* fast xfr by default */
void abortLog__fnc(char* msg, char* file, int line);
void abortLog__fnc(char* msg, char* file, int line) {
FILE* fp = fopen("CRASH.TXT", "wb");
if (!fp) fp = fopen("/tmp/CRASH.TXT", "wb");
if (!fp) fp = fopen("C:\\CRASH.TXT", "wb");
if (!fp) fp = fopen("CRASH.TXT", "wb");
void abortLog__fnc(char *msg, char *file, int line);
void abortLog__fnc(char *msg, char *file, int line) {
FILE *fp = fopen("CRASH.TXT", "wb");
if (!fp)
fp = fopen("/tmp/CRASH.TXT", "wb");
if (!fp)
fp = fopen("C:\\CRASH.TXT", "wb");
if (!fp)
fp = fopen("CRASH.TXT", "wb");
if (fp) {
fprintf(fp, "HTTrack " HTTRACK_VERSIONID " closed at '%s', line %d\r\n", file, line);
fprintf(fp, "HTTrack " HTTRACK_VERSIONID " closed at '%s', line %d\r\n",
file, line);
fprintf(fp, "Reason:\r\n%s\r\n", msg);
fflush(fp);
fclose(fp);
}
}
HTSEXT_API t_abortLog abortLog__ = abortLog__fnc; /* avoid VC++ inlining */
static void htspe_log(htsmoduleStruct* str, const char* msg);
HTSEXT_API t_abortLog abortLog__ = abortLog__fnc; /* avoid VC++ inlining */
int hts_parse_externals(htsmoduleStruct* str) {
static void htspe_log(htsmoduleStruct * str, const char *msg);
int hts_parse_externals(htsmoduleStruct * str) {
str->wrapper_name = "wrapper-lib";
/* External callback */
@@ -138,7 +146,8 @@ int hts_parse_externals(htsmoduleStruct* str) {
if (str->wrapper_name == NULL)
str->wrapper_name = "wrapper-lib";
/* Blacklisted */
if (multipleStringMatch(str->wrapper_name, StringBuff(str->opt->mod_blacklist))) {
if (multipleStringMatch
(str->wrapper_name, StringBuff(str->opt->mod_blacklist))) {
return -1;
} else {
htspe_log(str, str->wrapper_name);
@@ -162,13 +171,14 @@ int hts_parse_externals(htsmoduleStruct* str) {
// chain->moduleHandle = moduleHandle;
//}
void clearCallbacks(htscallbacks* chain_);
void clearCallbacks(htscallbacks* chain_) {
htscallbacks* chain;
void clearCallbacks(htscallbacks * chain_);
void clearCallbacks(htscallbacks * chain_) {
htscallbacks *chain;
chain = chain_;
while(chain != NULL) {
if (chain->exitFnc != NULL) {
(void) chain->exitFnc(); /* result ignored */
(void) chain->exitFnc(); /* result ignored */
chain->exitFnc = NULL;
}
chain = chain->next;
@@ -184,18 +194,20 @@ void clearCallbacks(htscallbacks* chain_) {
}
chain = chain->next;
}
chain = chain_->next; // Don't free the block #0
chain = chain_->next; // Don't free the block #0
while(chain != NULL) {
htscallbacks* nextchain = chain->next;
htscallbacks *nextchain = chain->next;
freet(chain);
chain = nextchain;
}
chain_->next = NULL; // Empty
chain_->next = NULL; // Empty
}
void* openFunctionLib(const char* file_) {
void* handle;
void *openFunctionLib(const char *file_) {
void *handle;
char *file = malloct(strlen(file_) + 32);
strcpy(file, file_);
#ifdef _WIN32
handle = LoadLibraryA(file);
@@ -214,7 +226,7 @@ void* openFunctionLib(const char* file_) {
return handle;
}
void closeFunctionLib(void* handle) {
void closeFunctionLib(void *handle) {
#ifdef _WIN32
FreeLibrary(handle);
#else
@@ -222,19 +234,20 @@ void closeFunctionLib(void* handle) {
#endif
}
void* getFunctionPtr(void* handle, const char* fncname_) {
void *getFunctionPtr(void *handle, const char *fncname_) {
if (handle) {
void* userfunction = NULL;
void *userfunction = NULL;
char *fncname = strdupt(fncname_);
/* Strip optional comma */
char *comma;
if ((comma = strchr(fncname, ',')) != NULL) { /* empty arg */
if ((comma = strchr(fncname, ',')) != NULL) { /* empty arg */
*comma++ = '\0';
}
/* the function itself */
userfunction = (void*) DynamicGet(handle, (char*)fncname);
userfunction = (void *) DynamicGet(handle, (char *) fncname);
freet(fncname);
@@ -243,34 +256,37 @@ void* getFunctionPtr(void* handle, const char* fncname_) {
return NULL;
}
void* ssl_handle = NULL;
void *ssl_handle = NULL;
#ifdef _WIN32
void* ssl_handle_2 = NULL;
void *ssl_handle_2 = NULL;
#endif
void htspe_init(void) {
static int initOk = 0;
if (!initOk) {
initOk = 1;
/* See CVE-2010-5252 */
#if (defined(_WIN32) && (!defined(_DEBUG)))
/* See KB 2389418
"If this parameter is an empty string (""), the call removes the
current directory from the default DLL search order" */
"If this parameter is an empty string (""), the call removes the
current directory from the default DLL search order" */
if (!SetDllDirectory("")) {
assertf(! "SetDllDirectory failed");
assertf(!"SetDllDirectory failed");
}
#endif
/* Zlib is now statically linked */
gz_is_available = 1;
/* OpenSSL */
#if HTS_DLOPEN
{
void* handle;
void *handle;
#ifdef _WIN32
handle = LoadLibraryA((char*)"ssleay32");
handle = LoadLibraryA((char *) "ssleay32");
#else
/* We are compatible with 0.9.6/7/8/8b and potentially above */
static const char *const libs[] = {
@@ -288,62 +304,82 @@ void htspe_init(void) {
"libssl.so.0.9.8n",
"libssl.so.0.9.8m",
"libssl.so.0.9.8l",
"libssl.so.0.9.8k", /* (Debarshi Ray) */
"libssl.so.0.9.8j", /* (Debarshi Ray) */
"libssl.so.0.9.8g", /* Added 8g release too (Debarshi Ray) */
"libssl.so.0.9.8k", /* (Debarshi Ray) */
"libssl.so.0.9.8j", /* (Debarshi Ray) */
"libssl.so.0.9.8g", /* Added 8g release too (Debarshi Ray) */
"libssl.so.0.9.8b",
"libssl.so.0.9.8",
"libssl.so.0.9.7",
"libssl.so.0.9.6",
"libssl.so", /* Try harder with devel link */
"libssl.so", /* Try harder with devel link */
NULL
};
int i;
for(i = 0, handle = NULL ; handle == NULL && libs[i] != NULL ; i++) {
for(i = 0, handle = NULL; handle == NULL && libs[i] != NULL; i++) {
handle = dlopen(libs[i], RTLD_LAZY);
}
#endif
ssl_handle = handle;
if (handle != NULL) {
SSL_shutdown = (t_SSL_shutdown) DynamicGet(handle, (char*)"SSL_shutdown");
SSL_free = (t_SSL_free) DynamicGet(handle, (char*)"SSL_free");
SSL_new = (t_SSL_new) DynamicGet(handle, (char*)"SSL_new");
SSL_clear = (t_SSL_clear) DynamicGet(handle, (char*)"SSL_clear");
SSL_set_fd = (t_SSL_set_fd) DynamicGet(handle, (char*)"SSL_set_fd");
SSL_set_connect_state = (t_SSL_set_connect_state) DynamicGet(handle, (char*)"SSL_set_connect_state");
SSL_connect = (t_SSL_connect) DynamicGet(handle, (char*)"SSL_connect");
SSL_get_error = (t_SSL_get_error) DynamicGet(handle, (char*)"SSL_get_error");
SSL_write = (t_SSL_write) DynamicGet(handle, (char*)"SSL_write");
SSL_read = (t_SSL_read) DynamicGet(handle, (char*)"SSL_read");
SSL_library_init = (t_SSL_library_init) DynamicGet(handle, (char*)"SSL_library_init");
ERR_load_SSL_strings = (t_ERR_load_SSL_strings) DynamicGet(handle, (char*)"ERR_load_SSL_strings");
SSLv23_client_method = (t_SSLv23_client_method) DynamicGet(handle, (char*)"SSLv23_client_method");
SSL_CTX_new = (t_SSL_CTX_new) DynamicGet(handle, (char*)"SSL_CTX_new");
SSL_load_error_strings = (t_SSL_load_error_strings) DynamicGet(handle, (char*)"SSL_load_error_strings");
SSL_CTX_ctrl = (t_SSL_CTX_ctrl) DynamicGet(handle, (char*)"SSL_CTX_ctrl");
SSL_shutdown =
(t_SSL_shutdown) DynamicGet(handle, (char *) "SSL_shutdown");
SSL_free = (t_SSL_free) DynamicGet(handle, (char *) "SSL_free");
SSL_new = (t_SSL_new) DynamicGet(handle, (char *) "SSL_new");
SSL_clear = (t_SSL_clear) DynamicGet(handle, (char *) "SSL_clear");
SSL_set_fd = (t_SSL_set_fd) DynamicGet(handle, (char *) "SSL_set_fd");
SSL_set_connect_state =
(t_SSL_set_connect_state) DynamicGet(handle,
(char *)
"SSL_set_connect_state");
SSL_connect =
(t_SSL_connect) DynamicGet(handle, (char *) "SSL_connect");
SSL_get_error =
(t_SSL_get_error) DynamicGet(handle, (char *) "SSL_get_error");
SSL_write = (t_SSL_write) DynamicGet(handle, (char *) "SSL_write");
SSL_read = (t_SSL_read) DynamicGet(handle, (char *) "SSL_read");
SSL_library_init =
(t_SSL_library_init) DynamicGet(handle, (char *) "SSL_library_init");
ERR_load_SSL_strings =
(t_ERR_load_SSL_strings) DynamicGet(handle,
(char *) "ERR_load_SSL_strings");
SSLv23_client_method =
(t_SSLv23_client_method) DynamicGet(handle,
(char *) "SSLv23_client_method");
SSL_CTX_new =
(t_SSL_CTX_new) DynamicGet(handle, (char *) "SSL_CTX_new");
SSL_load_error_strings =
(t_SSL_load_error_strings) DynamicGet(handle,
(char *)
"SSL_load_error_strings");
SSL_CTX_ctrl =
(t_SSL_CTX_ctrl) DynamicGet(handle, (char *) "SSL_CTX_ctrl");
#ifdef _WIN32
handle = LoadLibraryA((char*)"libeay32");
handle = LoadLibraryA((char *) "libeay32");
ssl_handle_2 = handle;
#endif
ERR_load_crypto_strings = (t_ERR_load_crypto_strings) DynamicGet(handle, (char*)"ERR_load_crypto_strings");
ERR_error_string = (t_ERR_error_string) DynamicGet(handle, (char*)"ERR_error_string");
ERR_load_crypto_strings =
(t_ERR_load_crypto_strings) DynamicGet(handle,
(char *)
"ERR_load_crypto_strings");
ERR_error_string =
(t_ERR_error_string) DynamicGet(handle, (char *) "ERR_error_string");
if (SSL_shutdown && SSL_free && SSL_CTX_ctrl && SSL_new && SSL_clear &&
SSL_set_fd && SSL_set_connect_state && SSL_connect && SSL_get_error && SSL_write
&& SSL_read && SSL_library_init && SSLv23_client_method && SSL_CTX_new
&& SSL_load_error_strings && ERR_error_string) {
if (SSL_shutdown && SSL_free && SSL_CTX_ctrl && SSL_new && SSL_clear
&& SSL_set_fd && SSL_set_connect_state && SSL_connect
&& SSL_get_error && SSL_write && SSL_read && SSL_library_init
&& SSLv23_client_method && SSL_CTX_new && SSL_load_error_strings
&& ERR_error_string) {
SSL_is_available = 1;
}
}
}
#endif
/* */
/* Options availability */
sprintf(WHAT_is_available, "%s%s%s",
V6_is_available ? "" : "-noV6",
gz_is_available ? "" : "-nozip",
SSL_is_available ? "" : "-nossl");
sprintf(WHAT_is_available, "%s%s%s", V6_is_available ? "" : "-noV6",
gz_is_available ? "" : "-nozip", SSL_is_available ? "" : "-nossl");
}
}
@@ -359,13 +395,14 @@ void htspe_uninit(void) {
#endif
}
static void htspe_log(htsmoduleStruct* str, const char* msg) {
const char* savename = str->filename;
httrackp* opt = (httrackp*) str->opt;
hts_log_print(opt, LOG_DEBUG, "(External module): parsing %s using module %s",
savename, msg);
static void htspe_log(htsmoduleStruct * str, const char *msg) {
const char *savename = str->filename;
httrackp *opt = (httrackp *) str->opt;
hts_log_print(opt, LOG_DEBUG, "(External module): parsing %s using module %s",
savename, msg);
}
HTSEXT_API const char* hts_is_available(void) {
HTSEXT_API const char *hts_is_available(void) {
return WHAT_is_available;
}

82
src/htsmodules.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsmodules.h subroutines: */
/* external modules (parsers) */
@@ -69,56 +66,56 @@ typedef struct hash_struct hash_struct;
#define HTS_DEF_FWSTRUCT_htsmoduleStruct
typedef struct htsmoduleStruct htsmoduleStruct;
#endif
typedef int (* t_htsAddLink)(htsmoduleStruct* str, char* link);
typedef int (*t_htsAddLink) (htsmoduleStruct * str, char *link);
/* Structure passed to the module */
struct htsmoduleStruct {
/* Read-only elements */
const char* filename; /* filename (C:\My Web Sites\...) */
int size; /* size of filename (should be > 0) */
const char* mime; /* MIME type of the object */
const char* url_host; /* incoming hostname (www.foo.com) */
const char* url_file; /* incoming filename (/bar/bar.gny) */
const char *filename; /* filename (C:\My Web Sites\...) */
int size; /* size of filename (should be > 0) */
const char *mime; /* MIME type of the object */
const char *url_host; /* incoming hostname (www.foo.com) */
const char *url_file; /* incoming filename (/bar/bar.gny) */
/* Write-only */
const char* wrapper_name; /* name of wrapper (static string) */
char* err_msg; /* if an error occured, the error message (max. 1KB) */
const char *wrapper_name; /* name of wrapper (static string) */
char *err_msg; /* if an error occured, the error message (max. 1KB) */
/* Read/Write */
int relativeToHtmlLink; /* set this to 1 if all urls you pass to addLink
are in fact relative to the html file where your
module was originally */
int relativeToHtmlLink; /* set this to 1 if all urls you pass to addLink
are in fact relative to the html file where your
module was originally */
/* Callbacks */
t_htsAddLink addLink; /* call this function when links are
being detected. it if not your responsability to decide
if the engine will keep them, or not. */
t_htsAddLink addLink; /* call this function when links are
being detected. it if not your responsability to decide
if the engine will keep them, or not. */
/* Optional */
char* localLink; /* if non null, the engine will write there the local
relative filename of the link added by addLink(), or
the absolute path if the link was refused by the wizard */
int localLinkSize; /* size of the optionnal buffer */
char *localLink; /* if non null, the engine will write there the local
relative filename of the link added by addLink(), or
the absolute path if the link was refused by the wizard */
int localLinkSize; /* size of the optionnal buffer */
/* User-defined */
void* userdef; /* can be used by callback routines
*/
void *userdef; /* can be used by callback routines
*/
/* The parser httrackp structure (may be used) */
httrackp* opt;
httrackp *opt;
/* Internal use - please don't touch */
lien_url** liens;
struct_back* sback;
cache_back* cache;
hash_struct* hashptr;
lien_url **liens;
struct_back *sback;
cache_back *cache;
hash_struct *hashptr;
int numero_passe;
int add_tab_alloc;
/* */
int* lien_tot_;
int* ptr_;
size_t* lien_size_;
char** lien_buffer_;
int *lien_tot_;
int *ptr_;
size_t *lien_size_;
char **lien_buffer_;
const char *page_charset_;
/* Internal use - please don't touch */
@@ -126,19 +123,20 @@ struct htsmoduleStruct {
/* Used to wrap module initialization */
/* return 1 if init was ok */
typedef int (*t_htsWrapperInit)(char *fn, char *args);
typedef int (*t_htsWrapperExit)(void);
typedef int (*t_htsWrapperPlugInit)(char *args);
typedef int (*t_htsWrapperInit) (char *fn, char *args);
typedef int (*t_htsWrapperExit) (void);
typedef int (*t_htsWrapperPlugInit) (char *args);
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
HTSEXT_API const char* hts_get_version_info(httrackp *opt);
HTSEXT_API const char* hts_is_available(void);
HTSEXT_API const char *hts_get_version_info(httrackp * opt);
HTSEXT_API const char *hts_is_available(void);
extern void htspe_init(void);
extern void htspe_uninit(void);
extern int hts_parse_externals(htsmoduleStruct* str);
extern int hts_parse_externals(htsmoduleStruct * str);
extern int gz_is_available;
/*extern int swf_is_available;*/
extern int SSL_is_available;
extern int V6_is_available;

2136
src/htsname.c
View File

File diff suppressed because it is too large Load Diff

61
src/htsname.h
View File

@@ -17,43 +17,40 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* savename routine (compute output filename) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSNAME_DEFH
#define HTSNAME_DEFH
#define HTSNAME_DEFH
#include "htsglobal.h"
#define DELAYED_EXT "delayed"
#define IS_DELAYED_EXT(a) ( ((a) != NULL) && ((a)[0] != 0) && strendwith_(a, "." DELAYED_EXT) )
HTS_STATIC int strendwith_(const char* a, const char* b) {
HTS_STATIC int strendwith_(const char *a, const char *b) {
int i, j;
for(i = 0 ; a[i] != 0 ; i++);
for(j = 0 ; b[j] != 0 ; j++);
for(i = 0; a[i] != 0; i++) ;
for(j = 0; b[j] != 0; j++) ;
while(i >= 0 && j >= 0 && a[i] == b[j]) {
i--;
j--;
}
return (j == -1);
}
#define CACHE_REFNAME "hts-cache/ref"
/* Library internal definictions */
@@ -86,33 +83,27 @@ typedef struct lien_back lien_back;
#endif
// note: 'headers' can either be null, or incomplete (only r member filled)
int url_savename(char* adr_complete, char* fil_complete, char* save,
char* former_adr, char* former_fil,
char* referer_adr, char* referer_fil,
httrackp* opt,
lien_url** liens, int lien_tot,
struct_back* sback,
cache_back* cache,
hash_struct* hash,
int ptr, int numero_passe,
const lien_back* headers);
int url_savename2(char* adr_complete, char* fil_complete, char* save,
char* former_adr, char* former_fil,
char* referer_adr, char* referer_fil,
httrackp* opt,
lien_url** liens, int lien_tot,
struct_back* sback,
cache_back* cache,
hash_struct* hash,
int ptr, int numero_passe,
const lien_back* headers,
const char *charset);
void standard_name(char* b,char* dot_pos,char* nom_pos,char* fil_complete,int short_ver);
void url_savename_addstr(char* d,char* s);
char* url_md5(char* digest_buffer, char* fil_complete);
int url_savename(char *adr_complete, char *fil_complete, char *save,
char *former_adr, char *former_fil, char *referer_adr,
char *referer_fil, httrackp * opt, lien_url ** liens,
int lien_tot, struct_back * sback, cache_back * cache,
hash_struct * hash, int ptr, int numero_passe,
const lien_back * headers);
int url_savename2(char *adr_complete, char *fil_complete, char *save,
char *former_adr, char *former_fil, char *referer_adr,
char *referer_fil, httrackp * opt, lien_url ** liens,
int lien_tot, struct_back * sback, cache_back * cache,
hash_struct * hash, int ptr, int numero_passe,
const lien_back * headers, const char *charset);
void standard_name(char *b, char *dot_pos, char *nom_pos, char *fil_complete,
int short_ver);
void url_savename_addstr(char *d, char *s);
char *url_md5(char *digest_buffer, char *fil_complete);
void url_savename_refname(const char *adr, const char *fil, char *filename);
char *url_savename_refname_fullpath(httrackp* opt, const char *adr, const char *fil);
void url_savename_refname_remove(httrackp* opt, const char *adr, const char *fil);
char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
const char *fil);
void url_savename_refname_remove(httrackp * opt, const char *adr,
const char *fil);
#endif
#endif

39
src/htsnet.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Net definitions */
/* Used in .c files that needs connect() functions and so */
@@ -47,32 +44,32 @@ Please visit our Website: http://www.httrack.com
#ifdef _WIN32
// pour read
#ifndef _WIN32_WCE
#include <io.h>
#include <io.h>
#endif
// pour FindFirstFile
#include <winbase.h>
#include <winbase.h>
#else
//typedef int T_SOC;
#define INVALID_SOCKET -1
#include <netdb.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/time.h>
#define INVALID_SOCKET -1
#include <netdb.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/time.h>
/* Force for sun env. */
#ifndef BSD_COMP
#define BSD_COMP
#endif
#include <sys/ioctl.h>
#ifndef BSD_COMP
#define BSD_COMP
#endif
#include <sys/ioctl.h>
/* gethostname & co */
#ifndef _WIN32
#include <unistd.h>
#endif
/* inet_addr */
#include <arpa/inet.h>
#include <arpa/inet.h>
// pas la peine normalement..
#ifndef HTS_DO_NOT_REDEFINE_in_addr_t
typedef unsigned long in_addr_t;
typedef unsigned long in_addr_t;
#endif
#endif
@@ -83,6 +80,7 @@ Please visit our Website: http://www.httrack.com
/* Ipv4 structures */
typedef struct in_addr INaddr;
/* This should handle all cases */
#ifndef HTS_DEF_FWSTRUCT_SOCaddr
#define HTS_DEF_FWSTRUCT_SOCaddr
@@ -117,7 +115,6 @@ struct SOCaddr {
server_len=sizeof(struct sockaddr_in); \
} while(0)
/* Copy sockaddr to another one */
#define SOCaddr_copyaddr(server, server_len, hpaddr, hpsize) do { \
if (hpsize == sizeof(struct sockaddr_in)) { \
@@ -157,6 +154,7 @@ strcpy(namebuf, dot); \
/* Ipv4 structures */
typedef struct in6_addr INaddr;
/* This should handle all cases */
#ifndef HTS_DEF_FWSTRUCT_SOCaddr
#define HTS_DEF_FWSTRUCT_SOCaddr
@@ -257,7 +255,7 @@ typedef struct t_fullhostent t_fullhostent;
#endif
struct t_fullhostent {
t_hostent hp;
char* list[2];
char *list[2];
char addr[HTS_MAXADDRLEN]; /* various struct sockaddr structures */
unsigned int addr_maxlen;
};
@@ -271,7 +269,4 @@ memset((h), 0, sizeof(t_fullhostent)); \
(h)->addr_maxlen = HTS_MAXADDRLEN; \
} while(0)
#endif

337
src/htsopt.h
View File

@@ -17,25 +17,21 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: HTTrack parameters block */
/* Called by httrack.h and some other files */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTTRACK_DEFOPT
#define HTTRACK_DEFOPT
@@ -72,7 +68,7 @@ typedef struct String String;
#ifndef HTS_DEF_STRUCT_String
#define HTS_DEF_STRUCT_String
struct String {
char* buffer_;
char *buffer_;
size_t length_;
size_t capacity_;
};
@@ -91,8 +87,8 @@ struct t_proxy {
int active;
String name;
int port;
String bindhost; // bind this host
};
String bindhost; // bind this host
};
/* Structure utile pour copier en bloc les paramètres */
#ifndef HTS_DEF_FWSTRUCT_htsfilters
@@ -100,18 +96,18 @@ struct t_proxy {
typedef struct htsfilters htsfilters;
#endif
struct htsfilters {
char*** filters;
int* filptr;
char ***filters;
int *filptr;
//int* filter_max;
};
/* User callbacks chain */
typedef int (*htscallbacksfncptr)(void);
typedef int (*htscallbacksfncptr) (void);
typedef struct htscallbacks htscallbacks;
struct htscallbacks {
void* moduleHandle;
void *moduleHandle;
htscallbacksfncptr exitFnc;
htscallbacks * next;
htscallbacks *next;
};
/* filenote() internal file structure */
@@ -120,8 +116,8 @@ struct htscallbacks {
typedef struct filenote_strc filenote_strc;
#endif
struct filenote_strc {
FILE* lst;
char path[STRING_SIZE*2];
FILE *lst;
char path[STRING_SIZE * 2];
};
/* concat() functions */
@@ -131,7 +127,7 @@ typedef struct concat_strc concat_strc;
#endif
struct concat_strc {
int index;
char buff[16][STRING_SIZE*2*2];
char buff[16][STRING_SIZE * 2 * 2];
};
/* int2 functions */
@@ -140,10 +136,10 @@ struct concat_strc {
typedef struct strc_int2bytes2 strc_int2bytes2;
#endif
struct strc_int2bytes2 {
char catbuff[CATBUFF_SIZE];
char catbuff[CATBUFF_SIZE];
char buff1[256];
char buff2[32];
char* buffadr[2];
char *buffadr[2];
};
/* cmd callback */
@@ -170,13 +166,13 @@ struct fspc_strc {
#ifndef HTS_DEF_DEFSTRUCT_hts_log_type
#define HTS_DEF_DEFSTRUCT_hts_log_type
typedef enum hts_log_type {
LOG_DEBUG,
LOG_INFO,
LOG_NOTICE,
LOG_WARNING,
LOG_ERROR,
LOG_PANIC,
LOG_ERRNO = 1 << 8
LOG_DEBUG,
LOG_INFO,
LOG_NOTICE,
LOG_WARNING,
LOG_ERROR,
LOG_PANIC,
LOG_ERRNO = 1 << 8
} hts_log_type;
#endif
@@ -208,7 +204,7 @@ typedef struct struct_inthash struct_inthash, *inthash;
typedef struct htsoptstate htsoptstate;
#endif
struct htsoptstate {
htsmutex lock; /* 3.41 */
htsmutex lock; /* 3.41 */
/* */
int stop;
int exit_xh;
@@ -216,32 +212,32 @@ struct htsoptstate {
/* */
int mimehtml_created;
String mimemid;
FILE* mimefp;
FILE *mimefp;
int delayedId;
/* */
filenote_strc strc;
/* Functions context (avoir thread variables!) */
filenote_strc strc;
/* Functions context (avoir thread variables!) */
htscallbacks callbacks;
concat_strc concat;
usercommand_strc usercmd;
fspc_strc fspc;
char *userhttptype;
int verif_backblue_done;
int verif_external_status;
t_dnscache *dns_cache;
/* HTML parsing state */
char _hts_errmsg[HTS_CDLMAXSIZE + 256];
int _hts_in_html_parsing;
int _hts_in_html_done;
int _hts_in_html_poll;
int _hts_setpause;
int _hts_in_mirror;
char** _hts_addurl;
concat_strc concat;
usercommand_strc usercmd;
fspc_strc fspc;
char *userhttptype;
int verif_backblue_done;
int verif_external_status;
t_dnscache *dns_cache;
/* HTML parsing state */
char _hts_errmsg[HTS_CDLMAXSIZE + 256];
int _hts_in_html_parsing;
int _hts_in_html_done;
int _hts_in_html_poll;
int _hts_setpause;
int _hts_in_mirror;
char **_hts_addurl;
int _hts_cancel;
htsoptstatecancel *cancel; /* 3.41 */
char HTbuff[2048];
htsoptstatecancel *cancel; /* 3.41 */
char HTbuff[2048];
unsigned int debug_state;
unsigned int tmpnameid; /* 3.41 */
unsigned int tmpnameid; /* 3.41 */
};
/* Library handles */
@@ -264,11 +260,11 @@ struct htslibhandles {
/* Javascript parser flags */
typedef enum htsparsejava_flags {
HTSPARSE_NONE = 0, // don't parse
HTSPARSE_DEFAULT = 1, // parse default (all)
HTSPARSE_NO_CLASS = 2, // don't parse .java
HTSPARSE_NO_JAVASCRIPT = 4, // don't parse .js
HTSPARSE_NO_AGGRESSIVE = 8 // don't aggressively parse .js or .java
HTSPARSE_NONE = 0, // don't parse
HTSPARSE_DEFAULT = 1, // parse default (all)
HTSPARSE_NO_CLASS = 2, // don't parse .java
HTSPARSE_NO_JAVASCRIPT = 4, // don't parse .js
HTSPARSE_NO_AGGRESSIVE = 8 // don't aggressively parse .js or .java
} htsparsejava_flags;
// paramètres httrack (options)
@@ -277,113 +273,113 @@ typedef enum htsparsejava_flags {
typedef struct httrackp httrackp;
#endif
struct httrackp {
size_t size_httrackp; // size of this structure
size_t size_httrackp; // size of this structure
/* */
int wizard; // wizard aucun/grand/petit
int flush; // fflush sur les fichiers log
int travel; // type de déplacements (same domain etc)
int seeker; // up & down
int depth; // nombre de niveaux de récursion
int extdepth; // nombre de niveaux de récursion à l'éxtérieur
int urlmode; // liens relatifs etc
int debug; // mode débug log
int getmode; // sauver html, images..
FILE* log; // fichier log
FILE* errlog; // et erreur
LLint maxsite; // taille max site
LLint maxfile_nonhtml; // taille max non html
LLint maxfile_html; // taille max html
int maxsoc; // nbre sockets
LLint fragment; // fragmentation d'un site
int nearlink; // prendre les images/data proche d'une page mais à l'extérieur
int makeindex; // faire un index
int kindex; // et un index 'keyword'
int delete_old; // effacer anciens fichiers
int timeout; // nombre de secondes de timeout
int rateout; // nombre d'octets minium pour le transfert
int maxtime; // temps max en secondes
int maxrate; // taux de transfert max
int mms_maxtime; // max duration of a mms file
float maxconn; // nombre max de connexions/s
int waittime; // démarrage programmé
int cache; // génération d'un cache
int wizard; // wizard aucun/grand/petit
int flush; // fflush sur les fichiers log
int travel; // type de déplacements (same domain etc)
int seeker; // up & down
int depth; // nombre de niveaux de récursion
int extdepth; // nombre de niveaux de récursion à l'éxtérieur
int urlmode; // liens relatifs etc
int debug; // mode débug log
int getmode; // sauver html, images..
FILE *log; // fichier log
FILE *errlog; // et erreur
LLint maxsite; // taille max site
LLint maxfile_nonhtml; // taille max non html
LLint maxfile_html; // taille max html
int maxsoc; // nbre sockets
LLint fragment; // fragmentation d'un site
int nearlink; // prendre les images/data proche d'une page mais à l'extérieur
int makeindex; // faire un index
int kindex; // et un index 'keyword'
int delete_old; // effacer anciens fichiers
int timeout; // nombre de secondes de timeout
int rateout; // nombre d'octets minium pour le transfert
int maxtime; // temps max en secondes
int maxrate; // taux de transfert max
int mms_maxtime; // max duration of a mms file
float maxconn; // nombre max de connexions/s
int waittime; // démarrage programmé
int cache; // génération d'un cache
//int aff_progress; // barre de progression
int shell; // gestion d'un shell par pipe stdin/stdout
t_proxy proxy; // configuration du proxy
int savename_83; // conversion 8-3 pour les noms de fichiers
int savename_type; // type de noms: structure originale/html-images en un seul niveau
String savename_userdef; // structure userdef (ex: %h%p/%n%q.%t)
int savename_delayed; // delayed type check
int delayed_cached; // delayed type check can be cached to speedup updates
int mimehtml; // MIME-html
int user_agent_send; // user agent (ex: httrack/1.0 [sun])
String user_agent; //
String referer; // referer
String from; // from
String path_log; // chemin pour cache et log
String path_html; // chemin pour miroir
String path_html_utf8; // chemin pour miroir, UTF-8
String path_bin; // chemin pour templates
int retry; // nombre d'essais supplémentaires en cas d'échec
int makestat; // mettre à jour un fichier log de statistiques de transfert
int maketrack; // mettre à jour un fichier log de statistiques d'opérations
int parsejava; // parsing des classes java pour récupérer les class, gif & cie ; see htsparsejava_flags
int hostcontrol; // abandon d'un host trop lent etc.
int errpage; // générer une page d'erreur en cas de 404 etc.
int check_type; // si type inconnu (cgi,asp,/) alors tester lien (et gérer moved éventuellement)
int all_in_cache; // tout mettre en cache!
int robots; // traitement des robots
int external; // pages externes->pages d'erreur
int passprivacy; // pas de mot de pass dans les liens externes?
int includequery; // include la query-string
int mirror_first_page; // miroir des liens
String sys_com; // commande système
int sys_com_exec; // executer commande
int accept_cookie; // gestion des cookies
t_cookie* cookie;
int http10; // forcer http 1.0
int nokeepalive; // pas de keep-alive
int nocompression; // pas de compression
int sizehack; // forcer réponse "mis à jour" si taille identique
int urlhack; // force "url normalization" to avoid loops
int tolerant; // accepter content-length incorrect
int parseall; // essayer de tout parser (tags inconnus contenant des liens, par exemple)
int parsedebug; // débugger parser (debug!)
int norecatch; // ne pas reprendre les fichiers effacés localement par l'utilisateur
int verbosedisplay; // animation textuelle
String footer; // ligne d'infos
int maxcache; // maximum en mémoire au niveau du cache (backing)
int shell; // gestion d'un shell par pipe stdin/stdout
t_proxy proxy; // configuration du proxy
int savename_83; // conversion 8-3 pour les noms de fichiers
int savename_type; // type de noms: structure originale/html-images en un seul niveau
String savename_userdef; // structure userdef (ex: %h%p/%n%q.%t)
int savename_delayed; // delayed type check
int delayed_cached; // delayed type check can be cached to speedup updates
int mimehtml; // MIME-html
int user_agent_send; // user agent (ex: httrack/1.0 [sun])
String user_agent; //
String referer; // referer
String from; // from
String path_log; // chemin pour cache et log
String path_html; // chemin pour miroir
String path_html_utf8; // chemin pour miroir, UTF-8
String path_bin; // chemin pour templates
int retry; // nombre d'essais supplémentaires en cas d'échec
int makestat; // mettre à jour un fichier log de statistiques de transfert
int maketrack; // mettre à jour un fichier log de statistiques d'opérations
int parsejava; // parsing des classes java pour récupérer les class, gif & cie ; see htsparsejava_flags
int hostcontrol; // abandon d'un host trop lent etc.
int errpage; // générer une page d'erreur en cas de 404 etc.
int check_type; // si type inconnu (cgi,asp,/) alors tester lien (et gérer moved éventuellement)
int all_in_cache; // tout mettre en cache!
int robots; // traitement des robots
int external; // pages externes->pages d'erreur
int passprivacy; // pas de mot de pass dans les liens externes?
int includequery; // include la query-string
int mirror_first_page; // miroir des liens
String sys_com; // commande système
int sys_com_exec; // executer commande
int accept_cookie; // gestion des cookies
t_cookie *cookie;
int http10; // forcer http 1.0
int nokeepalive; // pas de keep-alive
int nocompression; // pas de compression
int sizehack; // forcer réponse "mis à jour" si taille identique
int urlhack; // force "url normalization" to avoid loops
int tolerant; // accepter content-length incorrect
int parseall; // essayer de tout parser (tags inconnus contenant des liens, par exemple)
int parsedebug; // débugger parser (debug!)
int norecatch; // ne pas reprendre les fichiers effacés localement par l'utilisateur
int verbosedisplay; // animation textuelle
String footer; // ligne d'infos
int maxcache; // maximum en mémoire au niveau du cache (backing)
//int maxcache_anticipate; // maximum de liens à anticiper (majorant)
int ftp_proxy; // proxy http pour ftp
String filelist; // fichier liste URL à inclure
String urllist; // fichier liste de filtres à inclure
htsfilters filters; // contient les pointeurs pour les filtres
hash_struct* hash; // hash structure
robots_wizard* robotsptr; // robots ptr
String lang_iso; // en, fr ..
String mimedefs; // ext1=mimetype1\next2=mimetype2..
String mod_blacklist; // (3.41)
int convert_utf8; // filenames UTF-8 conversion (3.46)
int ftp_proxy; // proxy http pour ftp
String filelist; // fichier liste URL à inclure
String urllist; // fichier liste de filtres à inclure
htsfilters filters; // contient les pointeurs pour les filtres
hash_struct *hash; // hash structure
robots_wizard *robotsptr; // robots ptr
String lang_iso; // en, fr ..
String mimedefs; // ext1=mimetype1\next2=mimetype2..
String mod_blacklist; // (3.41)
int convert_utf8; // filenames UTF-8 conversion (3.46)
//
int maxlink; // nombre max de liens
int maxfilter; // nombre max de filtres
int maxlink; // nombre max de liens
int maxfilter; // nombre max de filtres
//
char* exec; // adresse du nom de l'éxecutable
char *exec; // adresse du nom de l'éxecutable
//
int quiet; // poser des questions autres que wizard?
int keyboard; // vérifier stdin
int bypass_limits; // bypass built-in limits
int background_on_suspend; // background process on suspend signal
int quiet; // poser des questions autres que wizard?
int keyboard; // vérifier stdin
int bypass_limits; // bypass built-in limits
int background_on_suspend; // background process on suspend signal
//
int is_update; // c'est une update (afficher "File updated...")
int dir_topindex; // reconstruire top index par la suite
int is_update; // c'est une update (afficher "File updated...")
int dir_topindex; // reconstruire top index par la suite
//
// callbacks
t_hts_htmlcheck_callbacks *callbacks_fun;
t_hts_htmlcheck_callbacks *callbacks_fun;
// store library handles
htslibhandles libHandles;
//
htsoptstate state; // state
htsoptstate state; // state
};
// stats for httrack
@@ -392,39 +388,38 @@ struct httrackp {
typedef struct hts_stat_struct hts_stat_struct;
#endif
struct hts_stat_struct {
LLint HTS_TOTAL_RECV; // flux entrant reçu
LLint stat_bytes; // octets écrits sur disque
LLint HTS_TOTAL_RECV; // flux entrant reçu
LLint stat_bytes; // octets écrits sur disque
// int HTS_TOTAL_RECV_STATE; // status: 0 tout va bien 1: ralentir un peu 2: ralentir 3: beaucoup
TStamp stat_timestart; // départ
TStamp stat_timestart; // départ
//
LLint total_packed; // flux entrant compressé reçu
LLint total_unpacked; // flux entrant compressé reçu
int total_packedfiles; // fichiers compressés
LLint total_packed; // flux entrant compressé reçu
LLint total_unpacked; // flux entrant compressé reçu
int total_packedfiles; // fichiers compressés
//
TStamp istat_timestart[2]; // départ pour calcul instantanné
LLint istat_bytes[2]; // calcul pour instantanné
TStamp istat_reference01; // top départ donné par #0 à #1
int istat_idlasttimer; // id du timer qui a récemment donné une stat
TStamp istat_timestart[2]; // départ pour calcul instantanné
LLint istat_bytes[2]; // calcul pour instantanné
TStamp istat_reference01; // top départ donné par #0 à #1
int istat_idlasttimer; // id du timer qui a récemment donné une stat
//
int stat_files; // nombre de fichiers écrits
int stat_updated_files; // nombre de fichiers mis à jour
int stat_background; // nombre de fichiers écrits en arrière plan
int stat_files; // nombre de fichiers écrits
int stat_updated_files; // nombre de fichiers mis à jour
int stat_background; // nombre de fichiers écrits en arrière plan
//
int stat_nrequests; // nombre de requêtes sur socket
int stat_sockid; // nombre de sockets allouées au total
int stat_nsocket; // nombre de sockets
int stat_errors; // nombre d'erreurs
int stat_errors_front; // idem, mais au tout premier niveau
int stat_warnings; // '' warnings
int stat_infos; // '' infos
int nbk; // fichiers anticipés en arrière plan et terminés
LLint nb; // données transférées actuellement (estimation)
int stat_nrequests; // nombre de requêtes sur socket
int stat_sockid; // nombre de sockets allouées au total
int stat_nsocket; // nombre de sockets
int stat_errors; // nombre d'erreurs
int stat_errors_front; // idem, mais au tout premier niveau
int stat_warnings; // '' warnings
int stat_infos; // '' infos
int nbk; // fichiers anticipés en arrière plan et terminés
LLint nb; // données transférées actuellement (estimation)
//
LLint rate;
//
TStamp last_connect; // last connect() call
TStamp last_request; // last request issued
TStamp last_connect; // last connect() call
TStamp last_request; // last request issued
};
#endif

69
src/htsparse.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsparse.h parser */
/* html/javascript/css parser */
@@ -58,29 +55,29 @@ typedef struct htsmoduleStructExtended htsmoduleStructExtended;
#endif
struct htsmoduleStructExtended {
/* Main object */
htsblk* r_;
htsblk *r_;
/* Error handling */
int* error_;
int* exit_xh_;
int* store_errpage_;
int *error_;
int *exit_xh_;
int *store_errpage_;
/* Structural */
int* filptr_;
char*** filters_;
robots_wizard* robots_;
hash_struct* hash_;
int* lien_max_;
int *filptr_;
char ***filters_;
robots_wizard *robots_;
hash_struct *hash_;
int *lien_max_;
/* Base & codebase */
char* base;
char* codebase;
char *base;
char *codebase;
/* Index */
int* makeindex_done_;
FILE** makeindex_fp_;
int* makeindex_links_;
char* makeindex_firstlink_;
int *makeindex_done_;
FILE **makeindex_fp_;
int *makeindex_links_;
char *makeindex_firstlink_;
/* Html templates */
char *template_header_;
@@ -88,21 +85,20 @@ struct htsmoduleStructExtended {
char *template_footer_;
/* Specific to downloads */
LLint* stat_fragment_;
LLint *stat_fragment_;
TStamp makestat_time;
FILE* makestat_fp;
LLint* makestat_total_;
int* makestat_lnk_;
FILE* maketrack_fp;
FILE *makestat_fp;
LLint *makestat_total_;
int *makestat_lnk_;
FILE *maketrack_fp;
/* Function-dependant */
char* loc_;
TStamp* last_info_shell_;
int* info_shell_;
char *loc_;
TStamp *last_info_shell_;
int *info_shell_;
};
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
@@ -110,7 +106,7 @@ struct htsmoduleStructExtended {
Main parser, attempt to scan links inside the html/css/js file
Parameters: The public module structure, and the private module variables
*/
int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre);
int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
/*
Check for 301,302.. errors ("moved") and handle them; re-isuue requests, make
@@ -118,30 +114,31 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre);
Parameters: The public module structure, and the private module variables
Returns 0 upon success
*/
int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre);
int hts_mirror_check_moved(htsmoduleStruct * str,
htsmoduleStructExtended * stre);
/*
Process user intercations: pause, add link, delete link..
*/
void hts_mirror_process_user_interaction(htsmoduleStruct* str, htsmoduleStructExtended* stre);
void hts_mirror_process_user_interaction(htsmoduleStruct * str,
htsmoduleStructExtended * stre);
/*
Get the next file on the queue, waiting for it, handling other files in background..
Parameters: The public module structure, and the private module variables
Returns 0 upon success
*/
int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* stre);
int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
htsmoduleStructExtended * stre);
/*
Wait for (adr, fil, save) to be started, that is,
to be ready for naming, having its header MIME type
If the final URL is to be forbidden, sets 'forbidden_url' to the corresponding value
*/
int hts_wait_delayed(htsmoduleStruct* str,
char* adr, char* fil, char* save,
char* parent_adr, char* parent_fil,
char* former_adr, char* former_fil,
int* forbidden_url);
int hts_wait_delayed(htsmoduleStruct * str, char *adr, char *fil, char *save,
char *parent_adr, char *parent_fil, char *former_adr,
char *former_fil, int *forbidden_url);
/* Context state */

59
src/htsrobots.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* robots.txt (website robot file) */
@@ -49,53 +46,55 @@ Please visit our Website: http://www.httrack.com
// -- robots --
// fil="" : vérifier si règle déja enregistrée
int checkrobots(robots_wizard* robots,char* adr,char* fil) {
int checkrobots(robots_wizard * robots, char *adr, char *fil) {
while(robots) {
if (strfield2(robots->adr,adr)) {
if (strfield2(robots->adr, adr)) {
if (fil[0]) {
int ptr=0;
int ptr = 0;
char line[250];
if (strnotempty(robots->token)) {
do {
ptr+=binput(robots->token+ptr,line,200);
if (line[0]=='/') { // absolu
if (strfield(fil,line)) { // commence avec ligne
return -1; // interdit
ptr += binput(robots->token + ptr, line, 200);
if (line[0] == '/') { // absolu
if (strfield(fil, line)) { // commence avec ligne
return -1; // interdit
}
} else { // relatif
if (strstrcase(fil,line)) {
} else { // relatif
if (strstrcase(fil, line)) {
return -1;
}
}
} while( (strnotempty(line)) && (ptr<(int) strlen(robots->token)) );
} while((strnotempty(line)) && (ptr < (int) strlen(robots->token)));
}
} else {
return -1;
}
}
robots=robots->next;
robots = robots->next;
}
return 0;
}
int checkrobots_set(robots_wizard* robots,char* adr,char* data) {
if (((int) strlen(adr)) >= sizeof(robots->adr) - 2) return 0;
if (((int) strlen(data)) >= sizeof(robots->token) - 2) return 0;
int checkrobots_set(robots_wizard * robots, char *adr, char *data) {
if (((int) strlen(adr)) >= sizeof(robots->adr) - 2)
return 0;
if (((int) strlen(data)) >= sizeof(robots->token) - 2)
return 0;
while(robots) {
if (strfield2(robots->adr,adr)) { // entrée existe
strcpybuff(robots->token,data);
if (strfield2(robots->adr, adr)) { // entrée existe
strcpybuff(robots->token, data);
#if DEBUG_ROBOTS
printf("robots.txt: set %s to %s\n",adr,data);
printf("robots.txt: set %s to %s\n", adr, data);
#endif
return -1;
}
else if (!robots->next) {
robots->next=(robots_wizard*) calloct(1,sizeof(robots_wizard));
} else if (!robots->next) {
robots->next = (robots_wizard *) calloct(1, sizeof(robots_wizard));
if (robots->next) {
robots->next->next=NULL;
strcpybuff(robots->next->adr,adr);
strcpybuff(robots->next->token,data);
robots->next->next = NULL;
strcpybuff(robots->next->adr, adr);
strcpybuff(robots->next->token, data);
#if DEBUG_ROBOTS
printf("robots.txt: new set %s to %s\n",adr,data);
printf("robots.txt: new set %s to %s\n", adr, data);
#endif
}
#if DEBUG_ROBOTS
@@ -103,15 +102,15 @@ int checkrobots_set(robots_wizard* robots,char* adr,char* data) {
printf("malloc error!!\n");
#endif
}
robots=robots->next;
robots = robots->next;
}
return 0;
}
void checkrobots_free(robots_wizard* robots) {
void checkrobots_free(robots_wizard * robots) {
if (robots->next) {
checkrobots_free(robots->next);
freet(robots->next);
robots->next=NULL;
robots->next = NULL;
}
}

15
src/htsrobots.h
View File

@@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
@@ -34,10 +32,8 @@ Please visit our Website: http://www.httrack.com
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSROBOTS_DEFH
#define HTSROBOTS_DEFH
#define HTSROBOTS_DEFH
// robots wizard
#ifndef HTS_DEF_FWSTRUCT_robots_wizard
@@ -47,15 +43,14 @@ typedef struct robots_wizard robots_wizard;
struct robots_wizard {
char adr[128];
char token[4096];
struct robots_wizard* next;
struct robots_wizard *next;
};
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
int checkrobots(robots_wizard* robots,char* adr,char* fil);
void checkrobots_free(robots_wizard* robots);
int checkrobots_set(robots_wizard* robots,char* adr,char* data);
int checkrobots(robots_wizard * robots, char *adr, char *fil);
void checkrobots_free(robots_wizard * robots);
int checkrobots_set(robots_wizard * robots, char *adr, char *data);
#endif
#endif

1100
src/htsserver.c
View File

File diff suppressed because it is too large Load Diff

235
src/htsserver.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Mini-server */
/* Author: Xavier Roche */
@@ -37,19 +34,18 @@ Please visit our Website: http://www.httrack.com
// Fichier intercepteur d'URL .h
#ifndef HTS_SERVER_DEFH
#define HTS_SERVER_DEFH
#define HTS_SERVER_DEFH
#include "htsnet.h"
/* String */
#include "htsstrings.h"
// Fonctions
void socinput(T_SOC soc,char* s,int max);
T_SOC smallserver_init_std(int* port_prox,char* adr_prox,int defaultPort);
T_SOC smallserver_init(int* port,char* adr);
int smallserver(T_SOC soc,char* url,char* method,char* data, char* path);
void socinput(T_SOC soc, char *s, int max);
T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort);
T_SOC smallserver_init(int *port, char *adr);
int smallserver(T_SOC soc, char *url, char *method, char *data, char *path);
#define CATCH_RESPONSE \
"HTTP/1.0 200 OK\r\n"\
@@ -90,67 +86,76 @@ extern httrackp *global_opt;
#define is_quote(c) ( ((c)=='\"') || ((c)=='\'') )
#define is_retorsep(c) ( ((c)==10) || ((c)==13) || ((c)==9) )
extern int smallserver_setkey(char* key, char* value);
extern int smallserver_setkeyint(char* key, LLint value);
extern int smallserver_setkeyarr(char* key, int id, char* key2, char* value);
extern int smallserver_setkey(char *key, char *value);
extern int smallserver_setkeyint(char *key, LLint value);
extern int smallserver_setkeyarr(char *key, int id, char *key2, char *value);
/* Language files */
static int htslang_load(char* limit_to, char* apppath);
static void conv_printf(char* from,char* to);
static int htslang_load(char *limit_to, char *apppath);
static void conv_printf(char *from, char *to);
static void LANG_DELETE(void);
static void LANG_INIT(char* path);
static int LANG_T(char* path, int l);
static void LANG_INIT(char *path);
static int LANG_T(char *path, int l);
static int QLANG_T(int l);
static char* LANGSEL(char* name);
static char* LANGINTKEY(char* name);
static int LANG_SEARCH(char* path, char* iso);
static int LANG_LIST(char* path, char* buffer);
static char *LANGSEL(char *name);
static char *LANGINTKEY(char *name);
static int LANG_SEARCH(char *path, char *iso);
static int LANG_LIST(char *path, char *buffer);
int htslang_init(void);
int htslang_uninit(void);
/* Static definitions */
static char* gethomedir(void);
static int linput_cpp(FILE* fp,char* s,int max);
static int linput_trim(FILE* fp,char* s,int max);
static int fexist(const char* s);
static int linput(FILE* fp,char* s,int max);
static char *gethomedir(void);
static int linput_cpp(FILE * fp, char *s, int max);
static int linput_trim(FILE * fp, char *s, int max);
static int fexist(const char *s);
static int linput(FILE * fp, char *s, int max);
static int linputsoc(T_SOC soc, char* s, int max) {
static int linputsoc(T_SOC soc, char *s, int max) {
int c;
int j=0;
int j = 0;
do {
unsigned char ch;
if (recv(soc, &ch, 1, 0) == 1) {
c = ch;
} else {
c = EOF;
}
if (c!=EOF) {
switch(c) {
case 13: break; // sauter CR
case 10: c=-1; break;
case 9: case 12: break; // sauter ces caractères
default: s[j++]=(char) c; break;
if (c != EOF) {
switch (c) {
case 13:
break; // sauter CR
case 10:
c = -1;
break;
case 9:
case 12:
break; // sauter ces caractères
default:
s[j++] = (char) c;
break;
}
}
} while((c!=-1) && (c!=EOF) && (j<(max-1)));
s[j]='\0';
} while((c != -1) && (c != EOF) && (j < (max - 1)));
s[j] = '\0';
return j;
}
static int check_readinput_t(T_SOC soc, int timeout) {
if (soc != INVALID_SOCKET) {
fd_set fds; // poll structures
fd_set fds; // poll structures
struct timeval tv; // structure for select
FD_ZERO(&fds);
FD_SET(soc,&fds);
tv.tv_sec=timeout;
tv.tv_usec=0;
select(soc + 1,&fds,NULL,NULL,&tv);
if (FD_ISSET(soc,&fds))
FD_SET(soc, &fds);
tv.tv_sec = timeout;
tv.tv_usec = 0;
select(soc + 1, &fds, NULL, NULL, &tv);
if (FD_ISSET(soc, &fds))
return 1;
else
return 0;
@@ -158,38 +163,42 @@ static int check_readinput_t(T_SOC soc, int timeout) {
return 0;
}
static int linputsoc_t(T_SOC soc, char* s, int max, int timeout) {
static int linputsoc_t(T_SOC soc, char *s, int max, int timeout) {
if (check_readinput_t(soc, timeout)) {
return linputsoc(soc, s, max);
}
return -1;
}
static char* gethomedir(void) {
char* home = getenv( "HOME" );
static char *gethomedir(void) {
char *home = getenv("HOME");
if (home)
return home;
else
return ".";
}
static int linput_cpp(FILE* fp,char* s,int max) {
int rlen=0;
s[0]='\0';
static int linput_cpp(FILE * fp, char *s, int max) {
int rlen = 0;
s[0] = '\0';
do {
int ret;
if (rlen>0)
if (s[rlen-1]=='\\')
s[--rlen]='\0'; // couper \ final
if (rlen > 0)
if (s[rlen - 1] == '\\')
s[--rlen] = '\0'; // couper \ final
// lire ligne
ret=linput_trim(fp,s+rlen,max-rlen);
if (ret>0)
rlen+=ret;
} while((s[max(rlen-1,0)]=='\\') && (rlen<max));
ret = linput_trim(fp, s + rlen, max - rlen);
if (ret > 0)
rlen += ret;
} while((s[max(rlen - 1, 0)] == '\\') && (rlen < max));
return rlen;
}
static int fexist(const char* s) {
static int fexist(const char *s) {
struct stat st;
memset(&st, 0, sizeof(st));
if (stat(s, &st) == 0) {
if (S_ISREG(st.st_mode)) {
@@ -197,45 +206,56 @@ static int fexist(const char* s) {
}
}
return 0;
}
static int linput(FILE* fp,char* s,int max) {
}
static int linput(FILE * fp, char *s, int max) {
int c;
int j=0;
int j = 0;
do {
c=fgetc(fp);
if (c!=EOF) {
switch(c) {
case 13: break; // sauter CR
case 10: c=-1; break;
case 0: case 9: case 12: break; // sauter ces caractères
default: s[j++]=(char) c; break;
c = fgetc(fp);
if (c != EOF) {
switch (c) {
case 13:
break; // sauter CR
case 10:
c = -1;
break;
case 0:
case 9:
case 12:
break; // sauter ces caractères
default:
s[j++] = (char) c;
break;
}
}
} while((c!=-1) && (c!=EOF) && (j<(max-1)));
s[j]='\0';
} while((c != -1) && (c != EOF) && (j < (max - 1)));
s[j] = '\0';
return j;
}
static int linput_trim(FILE* fp,char* s,int max) {
int rlen=0;
char* ls=(char*) malloct(max+2);
s[0]='\0';
static int linput_trim(FILE * fp, char *s, int max) {
int rlen = 0;
char *ls = (char *) malloct(max + 2);
s[0] = '\0';
if (ls) {
char* a;
char *a;
// lire ligne
rlen=linput(fp,ls,max);
rlen = linput(fp, ls, max);
if (rlen) {
// sauter espaces et tabs en fin
while( (rlen>0) && is_realspace(ls[max(rlen-1,0)]) )
ls[--rlen]='\0';
while((rlen > 0) && is_realspace(ls[max(rlen - 1, 0)]))
ls[--rlen] = '\0';
// sauter espaces en début
a=ls;
while((rlen>0) && ((*a==' ') || (*a=='\t'))) {
a = ls;
while((rlen > 0) && ((*a == ' ') || (*a == '\t'))) {
a++;
rlen--;
}
if (rlen>0) {
memcpy(s,a,rlen); // can copy \0 chars
s[rlen]='\0';
if (rlen > 0) {
memcpy(s, a, rlen); // can copy \0 chars
s[rlen] = '\0';
}
}
//
@@ -245,57 +265,60 @@ static int linput_trim(FILE* fp,char* s,int max) {
}
static int ehexh(char c) {
if ((c>='0') && (c<='9')) return c-'0';
if ((c>='a') && (c<='f')) c-=('a'-'A');
if ((c>='A') && (c<='F')) return (c-'A'+10);
if ((c >= '0') && (c <= '9'))
return c - '0';
if ((c >= 'a') && (c <= 'f'))
c -= ('a' - 'A');
if ((c >= 'A') && (c <= 'F'))
return (c - 'A' + 10);
return 0;
}
static int ehex(char* s) {
return 16*ehexh(*s)+ehexh(*(s+1));
static int ehex(char *s) {
return 16 * ehexh(*s) + ehexh(*(s + 1));
}
static void unescapehttp(char* s, String* tempo) {
static void unescapehttp(char *s, String * tempo) {
int i;
for (i=0;i<(int) strlen(s);i++) {
if (s[i]=='%' && s[i+1]=='%') {
for(i = 0; i < (int) strlen(s); i++) {
if (s[i] == '%' && s[i + 1] == '%') {
i++;
StringAddchar(*tempo, '%');
} else if (s[i]=='%') {
} else if (s[i] == '%') {
char hc;
i++;
hc = (char) ehex(s+i);
hc = (char) ehex(s + i);
StringAddchar(*tempo, (char) hc);
i++; // sauter 2 caractères finalement
}
else if (s[i]=='+') {
i++; // sauter 2 caractères finalement
} else if (s[i] == '+') {
StringAddchar(*tempo, ' ');
}
else
} else
StringAddchar(*tempo, s[i]);
}
}
static void unescapeini(char* s, String* tempo) {
static void unescapeini(char *s, String * tempo) {
int i;
char lastc=0;
for (i=0;i<(int) strlen(s);i++) {
if (s[i]=='%' && s[i+1]=='%') {
char lastc = 0;
for(i = 0; i < (int) strlen(s); i++) {
if (s[i] == '%' && s[i + 1] == '%') {
i++;
StringAddchar(*tempo, lastc = '%');
} else if (s[i]=='%') {
} else if (s[i] == '%') {
char hc;
i++;
hc = (char) ehex(s+i);
hc = (char) ehex(s + i);
if (!is_retorsep(hc) || !is_retorsep(lastc)) {
StringAddchar(*tempo, lastc = (char) hc);
}
i++; // sauter 2 caractères finalement
}
else
i++; // sauter 2 caractères finalement
} else
StringAddchar(*tempo, lastc = s[i]);
}
}
#endif

21
src/htsstrings.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Strings */
/* Author: Xavier Roche */
@@ -37,7 +34,7 @@ Please visit our Website: http://www.httrack.com
/* Safer Strings ; standalone .h library */
#ifndef HTS_STRINGS_DEFSTATIC
#define HTS_STRINGS_DEFSTATIC
#define HTS_STRINGS_DEFSTATIC
/* System definitions. */
#include <string.h>
@@ -63,7 +60,7 @@ typedef struct String String;
#ifndef HTS_DEF_STRUCT_String
#define HTS_DEF_STRUCT_String
struct String {
char* buffer_;
char *buffer_;
size_t length_;
size_t capacity_;
};
@@ -135,8 +132,8 @@ struct String {
#define StringRoom(BLK, SIZE) StringRoomTotal(BLK, StringLength(BLK) + (SIZE) + 1)
/** Return the RW buffer for a strcat() operation of at most SIZE characters. **/
#define StringBuffN(BLK, SIZE) StringBuffN_(&(BLK), SIZE)
HTS_STATIC char* StringBuffN_(String* blk, int size) {
#define StringBuffN(BLK, SIZE) StringBuffN_(&(BLK), SIZE)
HTS_STATIC char *StringBuffN_(String * blk, int size) {
StringRoom(*blk, size);
return StringBuffRW(*blk);
}
@@ -212,8 +209,9 @@ The pointer _MUST_ be compatible with STRING_REALLOC() and STRING_FREE() **/
} while(0)
/** Acquire a string ; it's the client's responsability to free() it **/
HTS_STATIC char* StringAcquire(String* blk) {
char* buff = StringBuffRW(*blk);
HTS_STATIC char *StringAcquire(String * blk) {
char *buff = StringBuffRW(*blk);
StringBuffRW(*blk) = NULL;
StringCapacity(*blk) = 0;
StringLength(*blk) = 0;
@@ -221,14 +219,15 @@ HTS_STATIC char* StringAcquire(String* blk) {
}
/** Clone a string. **/
HTS_STATIC String StringDup(const String* src) {
HTS_STATIC String StringDup(const String * src) {
String s = STRING_EMPTY;
StringMemcat(s, StringBuff(*src), StringLength(*src));
return s;
}
/** Attach a string using a pointer. **/
HTS_STATIC void StringAttach(String* blk, char** str) {
HTS_STATIC void StringAttach(String * blk, char **str) {
StringFree(*blk);
if (str != NULL && *str != NULL) {
StringBuffRW(*blk) = *str;

53
src/htsthread.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Threads */
/* Author: Xavier Roche */
@@ -50,16 +47,17 @@ Please visit our Website: http://www.httrack.com
static int process_chain = 0;
static htsmutex process_chain_mutex = HTSMUTEX_INIT;
HTSEXT_API void htsthread_wait(void ) {
HTSEXT_API void htsthread_wait(void) {
htsthread_wait_n(0);
}
HTSEXT_API void htsthread_wait_n(int n_wait) {
#if USE_BEGINTHREAD
int wait = 0;
do {
hts_mutexlock(&process_chain_mutex);
wait = (process_chain > n_wait );
wait = (process_chain > n_wait);
hts_mutexrelease(&process_chain_mutex);
if (wait)
Sleep(100);
@@ -68,7 +66,7 @@ HTSEXT_API void htsthread_wait_n(int n_wait) {
}
/* ensure initialized */
HTSEXT_API void htsthread_init(void ) {
HTSEXT_API void htsthread_init(void) {
#if USE_BEGINTHREAD
#if (defined(_DEBUG) || defined(DEBUG))
assertf(process_chain == 0);
@@ -79,7 +77,7 @@ HTSEXT_API void htsthread_init(void ) {
#endif
}
HTSEXT_API void htsthread_uninit(void ) {
HTSEXT_API void htsthread_uninit(void) {
htsthread_wait();
#if USE_BEGINTHREAD
hts_mutexfree(&process_chain_mutex);
@@ -88,18 +86,19 @@ HTSEXT_API void htsthread_uninit(void ) {
typedef struct hts_thread_s {
void *arg;
void (*fun)(void *arg);
void (*fun) (void *arg);
} hts_thread_s;
#ifdef _WIN32
static unsigned int __stdcall hts_entry_point(void *tharg)
#else
static void* hts_entry_point(void *tharg)
static void *hts_entry_point(void *tharg)
#endif
{
hts_thread_s *s_args = (hts_thread_s*) tharg;
void * const arg = s_args->arg;
void (*fun)(void *arg) = s_args->fun;
hts_thread_s *s_args = (hts_thread_s *) tharg;
void *const arg = s_args->arg;
void (*fun) (void *arg) = s_args->fun;
free(tharg);
hts_mutexlock(&process_chain_mutex);
@@ -122,16 +121,17 @@ static void* hts_entry_point(void *tharg)
}
/* create a thread */
HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg)
{
HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg) {
hts_thread_s *s_args = malloc(sizeof(hts_thread_s));
assertf(s_args != NULL);
s_args->arg = arg;
s_args->fun = fun;
#ifdef _WIN32
{
unsigned int idt;
HANDLE handle = (HANDLE) _beginthreadex(NULL, 0, hts_entry_point, s_args, 0, &idt);
HANDLE handle =
(HANDLE) _beginthreadex(NULL, 0, hts_entry_point, s_args, 0, &idt);
if (handle == 0) {
free(s_args);
return -1;
@@ -142,13 +142,15 @@ HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg)
}
#else
{
const size_t stackSize = 1024*1024*8;
const size_t stackSize = 1024 * 1024 * 8;
pthread_attr_t attr;
pthread_t handle = 0;
int retcode;
if (pthread_attr_init(&attr) != 0
|| pthread_attr_setstacksize(&attr, stackSize) != 0
|| (retcode = pthread_create(&handle, &attr, hts_entry_point, s_args)) != 0) {
|| pthread_attr_setstacksize(&attr, stackSize) != 0
|| (retcode =
pthread_create(&handle, &attr, hts_entry_point, s_args)) != 0) {
free(s_args);
return -1;
} else {
@@ -165,8 +167,9 @@ HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg)
/* Note: new 3.41 cleaned up functions. */
HTSEXT_API void hts_mutexinit(htsmutex* mutex) {
htsmutex_s* smutex = malloct(sizeof(htsmutex_s));
HTSEXT_API void hts_mutexinit(htsmutex * mutex) {
htsmutex_s *smutex = malloct(sizeof(htsmutex_s));
#ifdef _WIN32
smutex->handle = CreateMutex(NULL, FALSE, NULL);
#else
@@ -175,21 +178,21 @@ HTSEXT_API void hts_mutexinit(htsmutex* mutex) {
*mutex = smutex;
}
HTSEXT_API void hts_mutexfree(htsmutex* mutex) {
HTSEXT_API void hts_mutexfree(htsmutex * mutex) {
if (mutex != NULL && *mutex != NULL) {
#ifdef _WIN32
CloseHandle((*mutex)->handle);
#else
pthread_mutex_destroy(& ( (*mutex)->handle ) );
pthread_mutex_destroy(&((*mutex)->handle));
#endif
freet(*mutex);
*mutex = NULL;
}
}
HTSEXT_API void hts_mutexlock(htsmutex* mutex) {
HTSEXT_API void hts_mutexlock(htsmutex * mutex) {
assertf(mutex != NULL);
if (*mutex == HTSMUTEX_INIT) { /* must be initialized */
if (*mutex == HTSMUTEX_INIT) { /* must be initialized */
hts_mutexinit(mutex);
}
assertf(*mutex != NULL);
@@ -201,7 +204,7 @@ HTSEXT_API void hts_mutexlock(htsmutex* mutex) {
#endif
}
HTSEXT_API void hts_mutexrelease(htsmutex* mutex) {
HTSEXT_API void hts_mutexrelease(htsmutex * mutex) {
assertf(mutex != NULL && *mutex != NULL);
#ifdef _WIN32
assert((*mutex)->handle != NULL);

20
src/htsthread.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Threads */
/* Author: Xavier Roche */
@@ -64,24 +61,25 @@ typedef struct htsmutex_s htsmutex_s, *htsmutex;
struct htsmutex_s {
HANDLE handle;
};
#else /* #ifdef _WIN32 */
#else /* #ifdef _WIN32 */
struct htsmutex_s {
pthread_mutex_t handle;
};
#endif /* #ifdef _WIN32 */
#endif /* #ifdef _WIN32 */
/* Library internal definictions */
HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg);
HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg);
#ifndef HTTRACK_DEFLIB
HTSEXT_API void htsthread_wait(void );
HTSEXT_API void htsthread_wait(void);
#endif
HTSEXT_API void htsthread_wait_n(int n_wait);
/* Locking functions */
HTSEXT_API void hts_mutexinit(htsmutex* mutex);
HTSEXT_API void hts_mutexfree(htsmutex* mutex);
HTSEXT_API void hts_mutexlock(htsmutex* mutex);
HTSEXT_API void hts_mutexrelease(htsmutex* mutex);
HTSEXT_API void hts_mutexinit(htsmutex * mutex);
HTSEXT_API void hts_mutexfree(htsmutex * mutex);
HTSEXT_API void hts_mutexlock(htsmutex * mutex);
HTSEXT_API void hts_mutexrelease(htsmutex * mutex);
#ifdef HTS_INTERNAL_BYTECODE
/* Thread initialization */

973
src/htstools.c
View File

File diff suppressed because it is too large Load Diff

53
src/htstools.h
View File

@@ -17,27 +17,23 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* various tools (filename analyzing ..) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSTOOLS_DEFH
#define HTSTOOLS_DEFH
#define HTSTOOLS_DEFH
/* specific definitions */
#include "htsglobal.h"
@@ -50,20 +46,22 @@ typedef struct httrackp httrackp;
#ifndef HTS_DEF_FWSTRUCT_find_handle_struct
#define HTS_DEF_FWSTRUCT_find_handle_struct
typedef struct find_handle_struct find_handle_struct;
typedef find_handle_struct* find_handle;
typedef find_handle_struct *find_handle;
#endif
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
int ident_url_relatif(const char *lien, const char* urladr, const char* urlfil, char* adr, char* fil);
int lienrelatif(char* s,const char* link,const char* curr);
int link_has_authority(const char* lien);
int link_has_authorization(const char* lien);
void long_to_83(int mode,char* n83,char* save);
void longfile_to_83(int mode,char* n83,char* save);
HTS_INLINE int __rech_tageq(const char* adr,const char* s);
HTS_INLINE int __rech_tageqbegdigits(const char* adr,const char* s);
HTS_INLINE int rech_tageq_all(const char* adr, const char* s);
int ident_url_relatif(const char *lien, const char *urladr, const char *urlfil,
char *adr, char *fil);
int lienrelatif(char *s, const char *link, const char *curr);
int link_has_authority(const char *lien);
int link_has_authorization(const char *lien);
void long_to_83(int mode, char *n83, char *save);
void longfile_to_83(int mode, char *n83, char *save);
HTS_INLINE int __rech_tageq(const char *adr, const char *s);
HTS_INLINE int __rech_tageqbegdigits(const char *adr, const char *s);
HTS_INLINE int rech_tageq_all(const char *adr, const char *s);
#define rech_tageq(adr,s) \
( \
( (*((adr)-1)=='<') || (is_space(*((adr)-1))) ) ? \
@@ -85,30 +83,33 @@ HTS_INLINE int rech_tageq_all(const char* adr, const char* s);
: 0\
)
//HTS_INLINE int rech_tageq(const char* adr,const char* s);
HTS_INLINE int rech_sampletag(const char* adr,const char* s);
HTS_INLINE int rech_endtoken(const char* adr, const char** start);
HTS_INLINE int check_tag(char* from,const char* tag);
int verif_backblue(httrackp* opt, const char* base);
int verif_external(httrackp *opt,int nb,int test);
HTS_INLINE int rech_sampletag(const char *adr, const char *s);
HTS_INLINE int rech_endtoken(const char *adr, const char **start);
HTS_INLINE int check_tag(char *from, const char *tag);
int verif_backblue(httrackp * opt, const char *base);
int verif_external(httrackp * opt, int nb, int test);
int istoobig(httrackp *opt,LLint size,LLint maxhtml,LLint maxnhtml,char* type);
HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binpath);
int istoobig(httrackp * opt, LLint size, LLint maxhtml, LLint maxnhtml,
char *type);
HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
const char *binpath);
// Portable directory find functions
// Directory find functions
HTSEXT_API find_handle hts_findfirst(char* path);
HTSEXT_API find_handle hts_findfirst(char *path);
HTSEXT_API int hts_findnext(find_handle find);
HTSEXT_API int hts_findclose(find_handle find);
//
HTSEXT_API char* hts_findgetname(find_handle find);
HTSEXT_API char *hts_findgetname(find_handle find);
HTSEXT_API int hts_findgetsize(find_handle find);
HTSEXT_API int hts_findisdir(find_handle find);
HTSEXT_API int hts_findisfile(find_handle find);
HTSEXT_API int hts_findissystem(find_handle find);
#ifndef HTTRACK_DEFLIB
HTSEXT_API char* hts_getcategory(const char* filename);
HTSEXT_API char* hts_getcategories(char* path, int type);
HTSEXT_API char *hts_getcategory(const char *filename);
HTSEXT_API char *hts_getcategories(char *path, int type);
#endif
#endif

605
src/htsweb.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: webhttrack.c routines */
/* Author: Xavier Roche */
@@ -76,43 +73,44 @@ Please visit our Website: http://www.httrack.com
#ifdef _WIN32
#ifndef __cplusplus
// DOS
#include <process.h> /* _beginthread, _endthread */
#include <process.h> /* _beginthread, _endthread */
#endif
#else
#endif
static htsmutex refreshMutex = HTSMUTEX_INIT;
static int help_server(char* dest_path, int defaultPort);
static int help_server(char *dest_path, int defaultPort);
extern int commandRunning;
extern int commandEnd;
extern int commandReturn;
extern int commandEndRequested;
extern char* commandReturnMsg;
extern char* commandReturnCmdl;
extern char *commandReturnMsg;
extern char *commandReturnCmdl;
static void htsweb_sig_brpipe( int code ) {
static void htsweb_sig_brpipe(int code) {
/* ignore */
}
int main(int argc, char* argv[])
{
int main(int argc, char *argv[]) {
int i;
int ret = 0;
int defaultPort = 0;
printf("Initialzing the server..\n");
#ifdef _WIN32
{
WORD wVersionRequested; // requested version WinSock API
WORD wVersionRequested; // requested version WinSock API
WSADATA wsadata; // Windows Sockets API data
int stat;
wVersionRequested = 0x0101;
stat = WSAStartup( wVersionRequested, &wsadata );
stat = WSAStartup(wVersionRequested, &wsadata);
if (stat != 0) {
fprintf(stderr, "Winsock not found!\n");
return -1;
} else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) {
} else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) {
fprintf(stderr, "WINSOCK.DLL does not support version 1.1\n");
WSACleanup();
return -1;
@@ -122,7 +120,9 @@ int main(int argc, char* argv[])
if (argc < 2 || (argc % 2) != 0) {
fprintf(stderr, "** Warning: use the webhttrack frontend if available\n");
fprintf(stderr, "usage: %s [--port <port>] <path-to-html-root-dir> [key value [key value]..]\n", argv[0]);
fprintf(stderr,
"usage: %s [--port <port>] <path-to-html-root-dir> [key value [key value]..]\n",
argv[0]);
fprintf(stderr, "example: %s /usr/share/httrack/\n", argv[0]);
return 1;
}
@@ -173,37 +173,40 @@ int main(int argc, char* argv[])
smallserver_setkey("HTTRACK_AFF_VERSION", HTTRACK_AFF_VERSION);
{
char tmp[32];
sprintf(tmp, "%d", -1);
smallserver_setkey("HTS_PLATFORM", tmp);
}
smallserver_setkey("HTTRACK_WEB", HTTRACK_WEB);
smallserver_setkey("HTTRACK_WEB", HTTRACK_WEB);
/* protected session-id */
{
char buff[1024];
char digest[32 + 2];
srand((unsigned int)time(NULL));
sprintf(buff, "%d-%d", (int)time(NULL), (int)rand());
domd5mem(buff,strlen(buff),digest,1);
srand((unsigned int) time(NULL));
sprintf(buff, "%d-%d", (int) time(NULL), (int) rand());
domd5mem(buff, strlen(buff), digest, 1);
smallserver_setkey("sid", digest);
smallserver_setkey("_sid", digest);
}
/* set commandline keys */
for(i = 2 ; i < argc ; i += 2) {
if (strcmp(argv[i], "--port") == 0) {
if (sscanf(argv[i + 1], "%d", &defaultPort) != 1 || defaultPort < 0 || defaultPort >= 65535 ) {
fprintf(stderr, "couldn't set the port number to %s\n", argv[i + 1]);
return -1;
}
} else {
smallserver_setkey(argv[i], argv[i + 1]);
}
for(i = 2; i < argc; i += 2) {
if (strcmp(argv[i], "--port") == 0) {
if (sscanf(argv[i + 1], "%d", &defaultPort) != 1 || defaultPort < 0
|| defaultPort >= 65535) {
fprintf(stderr, "couldn't set the port number to %s\n", argv[i + 1]);
return -1;
}
} else {
smallserver_setkey(argv[i], argv[i + 1]);
}
}
/* sigpipe */
#ifndef _WIN32
signal( SIGPIPE , htsweb_sig_brpipe ); // broken pipe (write into non-opened socket)
signal(SIGPIPE, htsweb_sig_brpipe); // broken pipe (write into non-opened socket)
#endif
/* launch */
@@ -219,14 +222,15 @@ int main(int argc, char* argv[])
return ret;
}
static int webhttrack_runmain(httrackp *opt, int argc, char** argv);
static void back_launch_cmd( void* pP ) {
char* cmd = (char*) pP;
char** argv = (char**) malloct(1024 * sizeof(char*));
static int webhttrack_runmain(httrackp * opt, int argc, char **argv);
static void back_launch_cmd(void *pP) {
char *cmd = (char *) pP;
char **argv = (char **) malloct(1024 * sizeof(char *));
int argc = 0;
int i = 0;
int g = 0;
//
//
httrackp *opt;
/* copy commandline */
@@ -235,9 +239,9 @@ static void back_launch_cmd( void* pP ) {
commandReturnCmdl = strdup(cmd);
/* split */
argv[0]="webhttrack";
argv[1]=cmd;
argc++;
argv[0] = "webhttrack";
argv[1] = cmd;
argc++;
i = 0;
while(cmd[i]) {
if (cmd[i] == '\t' || cmd[i] == '\r' || cmd[i] == '\n') {
@@ -246,19 +250,20 @@ static void back_launch_cmd( void* pP ) {
i++;
}
i = 0;
while(cmd[i]) {
if(cmd[i]=='\"') g=!g;
if(cmd[i]==' ') {
if(!g){
cmd[i]='\0';
argv[argc++]=cmd+i+1;
while(cmd[i]) {
if (cmd[i] == '\"')
g = !g;
if (cmd[i] == ' ') {
if (!g) {
cmd[i] = '\0';
argv[argc++] = cmd + i + 1;
}
}
}
i++;
}
/* init */
hts_init();
/* init */
hts_init();
global_opt = opt = hts_create_opt();
/* run */
@@ -269,9 +274,9 @@ static void back_launch_cmd( void* pP ) {
commandReturnMsg = strdup(hts_errmsg(opt));
}
/* free */
global_opt = NULL;
hts_free_opt(opt);
/* free */
global_opt = NULL;
hts_free_opt(opt);
hts_uninit();
/* okay */
@@ -283,12 +288,12 @@ static void back_launch_cmd( void* pP ) {
/* free */
free(cmd);
freet(argv);
return ;
return;
}
void webhttrack_main(char* cmd) {
void webhttrack_main(char *cmd) {
commandRunning = 1;
hts_newthread(back_launch_cmd, (void*) strdup(cmd));
hts_newthread(back_launch_cmd, (void *) strdup(cmd));
}
void webhttrack_lock(void) {
@@ -299,8 +304,8 @@ void webhttrack_release(void) {
hts_mutexrelease(&refreshMutex);
}
static int webhttrack_runmain(httrackp *opt, int argc, char** argv) {
int ret;
static int webhttrack_runmain(httrackp * opt, int argc, char **argv) {
int ret;
CHAIN_FUNCTION(opt, init, htsshow_init, NULL);
CHAIN_FUNCTION(opt, uninit, htsshow_uninit, NULL);
@@ -326,37 +331,42 @@ static int webhttrack_runmain(httrackp *opt, int argc, char** argv) {
CHAIN_FUNCTION(opt, sendhead, htsshow_sendheader, NULL);
CHAIN_FUNCTION(opt, receivehead, htsshow_receiveheader, NULL);
ret = hts_main2(argc, argv, opt);
ret = hts_main2(argc, argv, opt);
htsthread_wait_n(1);
return ret;
return ret;
}
static int help_server(char* dest_path, int defaultPort) {
static int help_server(char *dest_path, int defaultPort) {
int returncode = 0;
char adr_prox[HTS_URLMAXSIZE*2];
char adr_prox[HTS_URLMAXSIZE * 2];
int port_prox;
T_SOC soc=smallserver_init_std(&port_prox, adr_prox, defaultPort);
if (soc!=INVALID_SOCKET) {
char url[HTS_URLMAXSIZE*2];
T_SOC soc = smallserver_init_std(&port_prox, adr_prox, defaultPort);
if (soc != INVALID_SOCKET) {
char url[HTS_URLMAXSIZE * 2];
char method[32];
char data[32768];
url[0]=method[0]=data[0]='\0';
url[0] = method[0] = data[0] = '\0';
//
printf("Okay, temporary server installed.\nThe URL is:\n");
printf("URL=http://%s:%d/\n", adr_prox, port_prox);
#ifndef _WIN32
{
pid_t pid = getpid();
printf("PID=%d\n", (int)pid);
printf("PID=%d\n", (int) pid);
}
#endif
fflush(stdout);
fflush(stderr);
//
if (!smallserver(soc,url,method,data,dest_path)) {
if (!smallserver(soc, url, method, data, dest_path)) {
int last_errno = errno;
fprintf(stderr, "Unable to create the server: %s\n", strerror(last_errno));
fprintf(stderr, "Unable to create the server: %s\n",
strerror(last_errno));
#ifdef _WIN32
closesocket(soc);
#else
@@ -368,7 +378,8 @@ static int help_server(char* dest_path, int defaultPort) {
returncode = 0;
}
} else {
fprintf(stderr, "Unable to initialize a temporary server (no remaining port)\n");
fprintf(stderr,
"Unable to initialize a temporary server (no remaining port)\n");
returncode = 1;
}
printf("EXITED\n");
@@ -377,50 +388,60 @@ static int help_server(char* dest_path, int defaultPort) {
return returncode;
}
/* CALLBACK FUNCTIONS */
/* Initialize the Winsock */
void __cdecl htsshow_init(t_hts_callbackarg *carg) {
void __cdecl htsshow_init(t_hts_callbackarg * carg) {
}
void __cdecl htsshow_uninit(t_hts_callbackarg *carg) {
void __cdecl htsshow_uninit(t_hts_callbackarg * carg) {
}
int __cdecl htsshow_start(t_hts_callbackarg *carg, httrackp* opt) {
return 1;
int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt) {
return 1;
}
int __cdecl htsshow_chopt(t_hts_callbackarg *carg, httrackp* opt) {
int __cdecl htsshow_chopt(t_hts_callbackarg * carg, httrackp * opt) {
return htsshow_start(carg, opt);
}
int __cdecl htsshow_end(t_hts_callbackarg *carg, httrackp* opt) {
return 1;
}
int __cdecl htsshow_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) {
int __cdecl htsshow_end(t_hts_callbackarg * carg, httrackp * opt) {
return 1;
}
int __cdecl htsshow_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) {
int __cdecl htsshow_preprocesshtml(t_hts_callbackarg * carg, httrackp * opt,
char **html, int *len,
const char *url_address,
const char *url_file) {
return 1;
}
int __cdecl htsshow_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file) {
int __cdecl htsshow_postprocesshtml(t_hts_callbackarg * carg, httrackp * opt,
char **html, int *len,
const char *url_address,
const char *url_file) {
return 1;
}
int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time, hts_stat_struct* stats) { // appelé à chaque boucle de HTTrack
static TStamp prev_mytime=0; /* ok */
static t_InpInfo SInfo; /* ok */
int __cdecl htsshow_checkhtml(t_hts_callbackarg * carg, httrackp * opt,
char *html, int len, const char *url_address,
const char *url_file) {
return 1;
}
int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * back, int back_max, int back_index, int lien_n, int lien_tot, int stat_time, hts_stat_struct * stats) { // appelé à chaque boucle de HTTrack
static TStamp prev_mytime = 0; /* ok */
static t_InpInfo SInfo; /* ok */
//
TStamp mytime;
long int rate=0;
long int rate = 0;
//
int stat_written=-1;
int stat_updated=-1;
int stat_errors=-1;
int stat_warnings=-1;
int stat_infos=-1;
int nbk=-1;
LLint nb=-1;
int stat_nsocket=-1;
LLint stat_bytes=-1;
LLint stat_bytes_recv=-1;
int irate=-1;
int stat_written = -1;
int stat_updated = -1;
int stat_errors = -1;
int stat_warnings = -1;
int stat_infos = -1;
int nbk = -1;
LLint nb = -1;
int stat_nsocket = -1;
LLint stat_bytes = -1;
LLint stat_bytes_recv = -1;
int irate = -1;
//
char st[256];
@@ -432,45 +453,57 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back
webhttrack_lock();
if (stats) {
stat_written=stats->stat_files;
stat_updated=stats->stat_updated_files;
stat_errors=stats->stat_errors;
stat_warnings=stats->stat_warnings;
stat_infos=stats->stat_infos;
nbk=stats->nbk;
stat_nsocket=stats->stat_nsocket;
irate=(int)stats->rate;
nb=stats->nb;
stat_bytes=stats->nb;
stat_bytes_recv=stats->HTS_TOTAL_RECV;
stat_written = stats->stat_files;
stat_updated = stats->stat_updated_files;
stat_errors = stats->stat_errors;
stat_warnings = stats->stat_warnings;
stat_infos = stats->stat_infos;
nbk = stats->nbk;
stat_nsocket = stats->stat_nsocket;
irate = (int) stats->rate;
nb = stats->nb;
stat_bytes = stats->nb;
stat_bytes_recv = stats->HTS_TOTAL_RECV;
}
mytime=mtime_local();
if ((stat_time>0) && (stat_bytes_recv>0))
rate=(int)(stat_bytes_recv/stat_time);
mytime = mtime_local();
if ((stat_time > 0) && (stat_bytes_recv > 0))
rate = (int) (stat_bytes_recv / stat_time);
else
rate=0; // pas d'infos
rate = 0; // pas d'infos
/* Infos */
if (stat_bytes>=0) SInfo.stat_bytes=stat_bytes; // bytes
if (stat_time>=0) SInfo.stat_time=stat_time; // time
if (lien_tot>=0) SInfo.lien_tot=lien_tot; // nb liens
if (lien_n>=0) SInfo.lien_n=lien_n; // scanned
SInfo.stat_nsocket=stat_nsocket; // socks
if (rate>0) SInfo.rate=rate; // rate
if (irate>=0) SInfo.irate=irate; // irate
if (SInfo.irate<0) SInfo.irate=SInfo.rate;
if (nbk>=0) SInfo.stat_back=nbk;
if (stat_written>=0) SInfo.stat_written=stat_written;
if (stat_updated>=0) SInfo.stat_updated=stat_updated;
if (stat_errors>=0) SInfo.stat_errors=stat_errors;
if (stat_warnings>=0) SInfo.stat_warnings=stat_warnings;
if (stat_infos>=0) SInfo.stat_infos=stat_infos;
st[0]='\0';
qsec2str(st,stat_time);
if (stat_bytes >= 0)
SInfo.stat_bytes = stat_bytes; // bytes
if (stat_time >= 0)
SInfo.stat_time = stat_time; // time
if (lien_tot >= 0)
SInfo.lien_tot = lien_tot; // nb liens
if (lien_n >= 0)
SInfo.lien_n = lien_n; // scanned
SInfo.stat_nsocket = stat_nsocket; // socks
if (rate > 0)
SInfo.rate = rate; // rate
if (irate >= 0)
SInfo.irate = irate; // irate
if (SInfo.irate < 0)
SInfo.irate = SInfo.rate;
if (nbk >= 0)
SInfo.stat_back = nbk;
if (stat_written >= 0)
SInfo.stat_written = stat_written;
if (stat_updated >= 0)
SInfo.stat_updated = stat_updated;
if (stat_errors >= 0)
SInfo.stat_errors = stat_errors;
if (stat_warnings >= 0)
SInfo.stat_warnings = stat_warnings;
if (stat_infos >= 0)
SInfo.stat_infos = stat_infos;
st[0] = '\0';
qsec2str(st, stat_time);
/* Set keys */
smallserver_setkeyint("info.stat_bytes", SInfo.stat_bytes);
smallserver_setkeyint("info.stat_time", SInfo.stat_time);
@@ -487,131 +520,148 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back
smallserver_setkeyint("info.stat_infos", SInfo.stat_infos);
/* */
smallserver_setkey("info.stat_time_str", st);
if ( ((mytime - prev_mytime)>100) || ((mytime - prev_mytime)<0) ) {
prev_mytime=mytime;
if (((mytime - prev_mytime) > 100) || ((mytime - prev_mytime) < 0)) {
prev_mytime = mytime;
// parcourir registre des liens
if (back_index>=0 && back_max > 0) { // seulement si index passé
int j,k;
int index=0;
int ok=0; // idem
int l; // idem
if (back_index >= 0 && back_max > 0) { // seulement si index passé
int j, k;
int index = 0;
int ok = 0; // idem
int l; // idem
//
t_StatsBuffer StatsBuffer[NStatsBuffer];
{
int i;
for(i=0;i<NStatsBuffer;i++) {
strcpybuff(StatsBuffer[i].state,"");
strcpybuff(StatsBuffer[i].name,"");
strcpybuff(StatsBuffer[i].file,"");
strcpybuff(StatsBuffer[i].url_sav,"");
StatsBuffer[i].back=0;
StatsBuffer[i].size=0;
StatsBuffer[i].sizetot=0;
for(i = 0; i < NStatsBuffer; i++) {
strcpybuff(StatsBuffer[i].state, "");
strcpybuff(StatsBuffer[i].name, "");
strcpybuff(StatsBuffer[i].file, "");
strcpybuff(StatsBuffer[i].url_sav, "");
StatsBuffer[i].back = 0;
StatsBuffer[i].size = 0;
StatsBuffer[i].sizetot = 0;
}
}
for(k=0;k<2;k++) { // 0: lien en cours 1: autres liens
for(j=0;(j<3) && (index<NStatsBuffer);j++) { // passe de priorité
for(k = 0; k < 2; k++) { // 0: lien en cours 1: autres liens
for(j = 0; (j < 3) && (index < NStatsBuffer); j++) { // passe de priorité
int _i;
for(_i=0+k;(_i< max(back_max*k,1) ) && (index<NStatsBuffer);_i++) { // no lien
int i=(back_index+_i)%back_max; // commencer par le "premier" (l'actuel)
if (back[i].status>=0) { // signifie "lien actif"
for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien
int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel)
if (back[i].status >= 0) { // signifie "lien actif"
// int ok=0; // OPTI
ok=0;
switch(j) {
case 0: // prioritaire
if ((back[i].status>0) && (back[i].status<99)) {
strcpybuff(StatsBuffer[index].state,"receive"); ok=1;
ok = 0;
switch (j) {
case 0: // prioritaire
if ((back[i].status > 0) && (back[i].status < 99)) {
strcpybuff(StatsBuffer[index].state, "receive");
ok = 1;
}
break;
case 1:
if (back[i].status==STATUS_WAIT_HEADERS) {
strcpybuff(StatsBuffer[index].state,"request"); ok=1;
if (back[i].status == STATUS_WAIT_HEADERS) {
strcpybuff(StatsBuffer[index].state, "request");
ok = 1;
} else if (back[i].status == STATUS_CONNECTING) {
strcpybuff(StatsBuffer[index].state, "connect");
ok = 1;
} else if (back[i].status == STATUS_WAIT_DNS) {
strcpybuff(StatsBuffer[index].state, "search");
ok = 1;
} else if (back[i].status == STATUS_FTP_TRANSFER) { // ohh le beau ftp
char proto[] = "ftp";
if (back[i].url_adr[0]) {
char *ep = strchr(back[i].url_adr, ':');
char *eps = strchr(back[i].url_adr, '/');
int count;
if (ep != NULL && ep < eps
&& (count = (int) (ep - back[i].url_adr)) < 4) {
proto[0] = '\0';
strncat(proto, back[i].url_adr, count);
}
}
sprintf(StatsBuffer[index].state, "%s: %s", proto,
back[i].info);
ok = 1;
}
else if (back[i].status==STATUS_CONNECTING) {
strcpybuff(StatsBuffer[index].state,"connect"); ok=1;
}
else if (back[i].status==STATUS_WAIT_DNS) {
strcpybuff(StatsBuffer[index].state,"search"); ok=1;
}
else if (back[i].status==STATUS_FTP_TRANSFER) { // ohh le beau ftp
char proto[] = "ftp";
if (back[i].url_adr[0]) {
char* ep = strchr(back[i].url_adr, ':');
char* eps = strchr(back[i].url_adr, '/');
int count;
if (ep != NULL && ep < eps && (count = (int) (ep - back[i].url_adr) ) < 4) {
proto[0] = '\0';
strncat(proto, back[i].url_adr, count);
}
}
sprintf(StatsBuffer[index].state,"%s: %s",proto,back[i].info); ok=1;
}
break;
default:
if (back[i].status==STATUS_READY) { // prêt
if ((back[i].r.statuscode==HTTP_OK)) {
strcpybuff(StatsBuffer[index].state,"ready"); ok=1;
}
else if ((back[i].r.statuscode>=100) && (back[i].r.statuscode<=599)) {
char tempo[256]; tempo[0]='\0';
infostatuscode(tempo,back[i].r.statuscode);
strcpybuff(StatsBuffer[index].state,tempo); ok=1;
}
else {
strcpybuff(StatsBuffer[index].state,"error"); ok=1;
break;
default:
if (back[i].status == STATUS_READY) { // prêt
if ((back[i].r.statuscode == HTTP_OK)) {
strcpybuff(StatsBuffer[index].state, "ready");
ok = 1;
} else if ((back[i].r.statuscode >= 100)
&& (back[i].r.statuscode <= 599)) {
char tempo[256];
tempo[0] = '\0';
infostatuscode(tempo, back[i].r.statuscode);
strcpybuff(StatsBuffer[index].state, tempo);
ok = 1;
} else {
strcpybuff(StatsBuffer[index].state, "error");
ok = 1;
}
}
break;
}
if (ok) {
char s[HTS_URLMAXSIZE*2];
char s[HTS_URLMAXSIZE * 2];
//
StatsBuffer[index].back=i; // index pour + d'infos
StatsBuffer[index].back = i; // index pour + d'infos
//
s[0]='\0';
strcpybuff(StatsBuffer[index].url_sav,back[i].url_sav); // pour cancel
if (strcmp(back[i].url_adr,"file://"))
strcatbuff(s,back[i].url_adr);
s[0] = '\0';
strcpybuff(StatsBuffer[index].url_sav, back[i].url_sav); // pour cancel
if (strcmp(back[i].url_adr, "file://"))
strcatbuff(s, back[i].url_adr);
else
strcatbuff(s,"localhost");
if (back[i].url_fil[0]!='/')
strcatbuff(s,"/");
strcatbuff(s,back[i].url_fil);
StatsBuffer[index].file[0]='\0';
strcatbuff(s, "localhost");
if (back[i].url_fil[0] != '/')
strcatbuff(s, "/");
strcatbuff(s, back[i].url_fil);
StatsBuffer[index].file[0] = '\0';
{
char* a=strrchr(s,'/');
char *a = strrchr(s, '/');
if (a) {
strncatbuff(StatsBuffer[index].file,a,200);
*a='\0';
strncatbuff(StatsBuffer[index].file, a, 200);
*a = '\0';
}
}
if ((l = (int) strlen(s))<MAX_LEN_INPROGRESS)
strcpybuff(StatsBuffer[index].name,s);
if ((l = (int) strlen(s)) < MAX_LEN_INPROGRESS)
strcpybuff(StatsBuffer[index].name, s);
else {
// couper
StatsBuffer[index].name[0]='\0';
strncatbuff(StatsBuffer[index].name,s,MAX_LEN_INPROGRESS/2-2);
strcatbuff(StatsBuffer[index].name,"...");
strcatbuff(StatsBuffer[index].name,s+l-MAX_LEN_INPROGRESS/2+2);
StatsBuffer[index].name[0] = '\0';
strncatbuff(StatsBuffer[index].name, s,
MAX_LEN_INPROGRESS / 2 - 2);
strcatbuff(StatsBuffer[index].name, "...");
strcatbuff(StatsBuffer[index].name,
s + l - MAX_LEN_INPROGRESS / 2 + 2);
}
if (back[i].r.totalsize>0) { // taille prédéfinie
StatsBuffer[index].sizetot=back[i].r.totalsize;
StatsBuffer[index].size=back[i].r.size;
} else { // pas de taille prédéfinie
if (back[i].status==STATUS_READY) { // prêt
StatsBuffer[index].sizetot=back[i].r.size;
StatsBuffer[index].size=back[i].r.size;
if (back[i].r.totalsize > 0) { // taille prédéfinie
StatsBuffer[index].sizetot = back[i].r.totalsize;
StatsBuffer[index].size = back[i].r.size;
} else { // pas de taille prédéfinie
if (back[i].status == STATUS_READY) { // prêt
StatsBuffer[index].sizetot = back[i].r.size;
StatsBuffer[index].size = back[i].r.size;
} else {
StatsBuffer[index].sizetot=8192;
StatsBuffer[index].size=(back[i].r.size % 8192);
StatsBuffer[index].sizetot = 8192;
StatsBuffer[index].size = (back[i].r.size % 8192);
}
}
index++;
@@ -623,20 +673,23 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back
/* Display current job */
{
int parsing=0;
int parsing = 0;
if (commandEndRequested)
smallserver_setkey("info.currentjob", "finishing pending transfers - Select [Cancel] to stop now!");
else if (!(parsing=hts_is_parsing(opt, -1)))
smallserver_setkey("info.currentjob",
"finishing pending transfers - Select [Cancel] to stop now!");
else if (!(parsing = hts_is_parsing(opt, -1)))
smallserver_setkey("info.currentjob", "receiving files");
else {
char tmp[1024];
tmp[0] = '\0';
switch(hts_is_testing(opt)) {
switch (hts_is_testing(opt)) {
case 0:
sprintf(tmp, "parsing HTML file (%d%%)",parsing);
sprintf(tmp, "parsing HTML file (%d%%)", parsing);
break;
case 1:
sprintf(tmp, "parsing HTML file: testing links (%d%%)",parsing);
sprintf(tmp, "parsing HTML file: testing links (%d%%)", parsing);
break;
case 2:
sprintf(tmp, "purging files");
@@ -658,72 +711,102 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back
/* Display background jobs */
{
int i;
for(i=0;i<NStatsBuffer;i++) {
for(i = 0; i < NStatsBuffer; i++) {
if (strnotempty(StatsBuffer[i].state)) {
strc_int2bytes2 strc;
strc_int2bytes2 strc;
smallserver_setkeyarr("info.state[", i, "]", StatsBuffer[i].state);
smallserver_setkeyarr("info.name[", i, "]", StatsBuffer[i].name);
smallserver_setkeyarr("info.file[", i, "]", StatsBuffer[i].file);
smallserver_setkeyarr("info.size[", i, "]", int2bytes(&strc,StatsBuffer[i].size));
smallserver_setkeyarr("info.sizetot[", i, "]", int2bytes(&strc,StatsBuffer[i].sizetot));
smallserver_setkeyarr("info.url_adr[", i, "]", StatsBuffer[i].url_adr);
smallserver_setkeyarr("info.url_fil[", i, "]", StatsBuffer[i].url_fil);
smallserver_setkeyarr("info.url_sav[", i, "]", StatsBuffer[i].url_sav);
smallserver_setkeyarr("info.size[", i, "]",
int2bytes(&strc, StatsBuffer[i].size));
smallserver_setkeyarr("info.sizetot[", i, "]",
int2bytes(&strc, StatsBuffer[i].sizetot));
smallserver_setkeyarr("info.url_adr[", i, "]",
StatsBuffer[i].url_adr);
smallserver_setkeyarr("info.url_fil[", i, "]",
StatsBuffer[i].url_fil);
smallserver_setkeyarr("info.url_sav[", i, "]",
StatsBuffer[i].url_sav);
}
}
}
}
}
}
/* UnLock */
webhttrack_release();
return 1;
}
const char* __cdecl htsshow_query(t_hts_callbackarg *carg, httrackp *opt, const char* question) {
static char s[]=""; /* ok */
const char *__cdecl htsshow_query(t_hts_callbackarg * carg, httrackp * opt,
const char *question) {
static char s[] = ""; /* ok */
return s;
}
const char* __cdecl htsshow_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question) {
static char s[]=""; /* ok */
const char *__cdecl htsshow_query2(t_hts_callbackarg * carg, httrackp * opt,
const char *question) {
static char s[] = ""; /* ok */
return s;
}
const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question) {
static char s[]=""; /* ok */
const char *__cdecl htsshow_query3(t_hts_callbackarg * carg, httrackp * opt,
const char *question) {
static char s[] = ""; /* ok */
return s;
}
int __cdecl htsshow_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status) {
int __cdecl htsshow_check(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil, int status) {
return -1;
}
int __cdecl htsshow_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status) {
int __cdecl htsshow_check_mime(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil,
const char *mime, int status) {
return -1;
}
void __cdecl htsshow_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile) {
void __cdecl htsshow_pause(t_hts_callbackarg * carg, httrackp * opt,
const char *lockfile) {
}
void __cdecl htsshow_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file) {
void __cdecl htsshow_filesave(t_hts_callbackarg * carg, httrackp * opt,
const char *file) {
}
void __cdecl htsshow_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated) {
void __cdecl htsshow_filesave2(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil,
const char *save, int is_new, int is_modified,
int not_updated) {
}
int __cdecl htsshow_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link) {
int __cdecl htsshow_linkdetected(t_hts_callbackarg * carg, httrackp * opt,
char *link) {
return 1;
}
int __cdecl htsshow_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag) {
int __cdecl htsshow_linkdetected2(t_hts_callbackarg * carg, httrackp * opt,
char *link, const char *start_tag) {
return 1;
}
int __cdecl htsshow_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, lien_back* back) {
int __cdecl htsshow_xfrstatus(t_hts_callbackarg * carg, httrackp * opt,
lien_back * back) {
return 1;
}
int __cdecl htsshow_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save) {
int __cdecl htsshow_savename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save) {
return 1;
}
int __cdecl htsshow_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing) {
int __cdecl htsshow_sendheader(t_hts_callbackarg * carg, httrackp * opt,
char *buff, const char *adr, const char *fil,
const char *referer_adr, const char *referer_fil,
htsblk * outgoing) {
return 1;
}
int __cdecl htsshow_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming) {
int __cdecl htsshow_receiveheader(t_hts_callbackarg * carg, httrackp * opt,
char *buff, const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil, htsblk * incoming) {
return 1;
}

95
src/htsweb.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: webhttrack.c routines */
/* Author: Xavier Roche */
@@ -47,16 +44,16 @@ typedef struct t_StatsBuffer {
char name[1024];
char file[1024];
char state[256];
char url_sav[HTS_URLMAXSIZE*2]; // pour cancel
char url_adr[HTS_URLMAXSIZE*2];
char url_fil[HTS_URLMAXSIZE*2];
char url_sav[HTS_URLMAXSIZE * 2]; // pour cancel
char url_adr[HTS_URLMAXSIZE * 2];
char url_fil[HTS_URLMAXSIZE * 2];
LLint size;
LLint sizetot;
int offset;
//
int back;
//
int actived; // pour disabled
int actived; // pour disabled
} t_StatsBuffer;
typedef struct t_InpInfo {
@@ -80,32 +77,66 @@ typedef struct t_InpInfo {
} t_InpInfo;
// wrappers
void __cdecl htsshow_init(t_hts_callbackarg *carg);
void __cdecl htsshow_uninit(t_hts_callbackarg *carg);
int __cdecl htsshow_start(t_hts_callbackarg *carg, httrackp* opt);
int __cdecl htsshow_chopt(t_hts_callbackarg *carg, httrackp* opt);
int __cdecl htsshow_end(t_hts_callbackarg *carg, httrackp* opt);
int __cdecl htsshow_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file);
int __cdecl htsshow_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file);
int __cdecl htsshow_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file);
int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats);
const char* __cdecl htsshow_query(t_hts_callbackarg *carg, httrackp *opt, const char* question);
const char* __cdecl htsshow_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question);
const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question);
int __cdecl htsshow_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status);
int __cdecl htsshow_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status);
void __cdecl htsshow_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile);
void __cdecl htsshow_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file);
void __cdecl htsshow_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified, int not_updated);
int __cdecl htsshow_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link);
int __cdecl htsshow_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag);
int __cdecl htsshow_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, lien_back* back);
int __cdecl htsshow_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save);
int __cdecl htsshow_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);
int __cdecl htsshow_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);
void __cdecl htsshow_init(t_hts_callbackarg * carg);
void __cdecl htsshow_uninit(t_hts_callbackarg * carg);
int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt);
int __cdecl htsshow_chopt(t_hts_callbackarg * carg, httrackp * opt);
int __cdecl htsshow_end(t_hts_callbackarg * carg, httrackp * opt);
int __cdecl htsshow_preprocesshtml(t_hts_callbackarg * carg, httrackp * opt,
char **html, int *len,
const char *url_address,
const char *url_file);
int __cdecl htsshow_postprocesshtml(t_hts_callbackarg * carg, httrackp * opt,
char **html, int *len,
const char *url_address,
const char *url_file);
int __cdecl htsshow_checkhtml(t_hts_callbackarg * carg, httrackp * opt,
char *html, int len, const char *url_address,
const char *url_file);
int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt,
lien_back * back, int back_max, int back_index,
int lien_n, int lien_tot, int stat_time,
hts_stat_struct * stats);
const char *__cdecl htsshow_query(t_hts_callbackarg * carg, httrackp * opt,
const char *question);
const char *__cdecl htsshow_query2(t_hts_callbackarg * carg, httrackp * opt,
const char *question);
const char *__cdecl htsshow_query3(t_hts_callbackarg * carg, httrackp * opt,
const char *question);
int __cdecl htsshow_check(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil, int status);
int __cdecl htsshow_check_mime(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil,
const char *mime, int status);
void __cdecl htsshow_pause(t_hts_callbackarg * carg, httrackp * opt,
const char *lockfile);
void __cdecl htsshow_filesave(t_hts_callbackarg * carg, httrackp * opt,
const char *file);
void __cdecl htsshow_filesave2(t_hts_callbackarg * carg, httrackp * opt,
const char *adr, const char *fil,
const char *save, int is_new, int is_modified,
int not_updated);
int __cdecl htsshow_linkdetected(t_hts_callbackarg * carg, httrackp * opt,
char *link);
int __cdecl htsshow_linkdetected2(t_hts_callbackarg * carg, httrackp * opt,
char *link, const char *start_tag);
int __cdecl htsshow_xfrstatus(t_hts_callbackarg * carg, httrackp * opt,
lien_back * back);
int __cdecl htsshow_savename(t_hts_callbackarg * carg, httrackp * opt,
const char *adr_complete, const char *fil_complete,
const char *referer_adr, const char *referer_fil,
char *save);
int __cdecl htsshow_sendheader(t_hts_callbackarg * carg, httrackp * opt,
char *buff, const char *adr, const char *fil,
const char *referer_adr, const char *referer_fil,
htsblk * outgoing);
int __cdecl htsshow_receiveheader(t_hts_callbackarg * carg, httrackp * opt,
char *buff, const char *adr, const char *fil,
const char *referer_adr,
const char *referer_fil, htsblk * incoming);
int main(int argc, char **argv);
void webhttrack_main(char* cmd);
void webhttrack_main(char *cmd);
void webhttrack_lock(void);
void webhttrack_release(void);

1145
src/htswizard.c
View File

File diff suppressed because it is too large Load Diff

24
src/htswizard.h
View File

@@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
@@ -34,9 +32,8 @@ Please visit our Website: http://www.httrack.com
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSWIZARD_DEFH
#define HTSWIZARD_DEFH
#define HTSWIZARD_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
@@ -53,19 +50,12 @@ typedef struct httrackp httrackp;
typedef struct lien_url lien_url;
#endif
int hts_acceptlink(httrackp* opt,
int ptr,int lien_tot,lien_url** liens,
char* adr,char* fil,
char* tag, char* attribute,
int* set_prio_to_0,
int* just_test_it);
int hts_testlinksize(httrackp* opt,
char* adr,char* fil,
LLint size);
int hts_acceptmime(httrackp* opt,
int ptr,int lien_tot,lien_url** liens,
char* adr,char* fil,
char* mime);
int hts_acceptlink(httrackp * opt, int ptr, int lien_tot, lien_url ** liens,
char *adr, char *fil, char *tag, char *attribute,
int *set_prio_to_0, int *just_test_it);
int hts_testlinksize(httrackp * opt, char *adr, char *fil, LLint size);
int hts_acceptmime(httrackp * opt, int ptr, int lien_tot, lien_url ** liens,
char *adr, char *fil, char *mime);
#endif
#endif

18
src/htswrap.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* wrapper system (for shell */
@@ -43,19 +40,20 @@ Please visit our Website: http://www.httrack.com
#include "htsinthash.h"
#include "htslib.h"
HTSEXT_API int htswrap_init(void) { // LEGACY
HTSEXT_API int htswrap_init(void) { // LEGACY
return 1;
}
HTSEXT_API int htswrap_free(void) { // LEGACY
HTSEXT_API int htswrap_free(void) { // LEGACY
return 1;
}
HTSEXT_API int htswrap_add(httrackp *opt, const char* name,void* fct) {
return hts_set_callback((t_hts_htmlcheck_callbacks*)opt->callbacks_fun, name, fct);
HTSEXT_API int htswrap_add(httrackp * opt, const char *name, void *fct) {
return hts_set_callback((t_hts_htmlcheck_callbacks *) opt->callbacks_fun,
name, fct);
}
HTSEXT_API uintptr_t htswrap_read(httrackp *opt, const char* name) {
return (uintptr_t) hts_get_callback((t_hts_htmlcheck_callbacks*)opt->callbacks_fun, name);
HTSEXT_API uintptr_t htswrap_read(httrackp * opt, const char *name) {
return (uintptr_t) hts_get_callback((t_hts_htmlcheck_callbacks *) opt->
callbacks_fun, name);
}

13
src/htswrap.h
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: httrack.c subroutines: */
/* wrapper system (for shell */
@@ -36,7 +33,7 @@ Please visit our Website: http://www.httrack.com
/* ------------------------------------------------------------ */
#ifndef HTSWRAP_DEFH
#define HTSWRAP_DEFH
#define HTSWRAP_DEFH
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
@@ -50,11 +47,11 @@ Please visit our Website: http://www.httrack.com
typedef struct httrackp httrackp;
#endif
HTSEXT_API int htswrap_init(void); // LEGACY
HTSEXT_API int htswrap_free(void); // LEGACY
HTSEXT_API int htswrap_init(void); // LEGACY
HTSEXT_API int htswrap_free(void); // LEGACY
HTSEXT_API int htswrap_add(httrackp *opt, const char* name, void* fct);
HTSEXT_API uintptr_t htswrap_read(httrackp *opt, const char* name);
HTSEXT_API int htswrap_add(httrackp * opt, const char *name, void *fct);
HTSEXT_API uintptr_t htswrap_read(httrackp * opt, const char *name);
#endif

130
src/htszlib.c
View File

@@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Unpacking subroutines using Jean-loup Gailly's Zlib */
/* for http compressed data */
@@ -55,30 +52,35 @@ Please visit our Website: http://www.httrack.com
Return value: size of the new file, or -1 if an error occured
*/
/* Note: utf-8 */
int hts_zunpack(char* filename,char* newfile) {
int hts_zunpack(char *filename, char *newfile) {
int ret = -1;
char catbuff[CATBUFF_SIZE];
char catbuff[CATBUFF_SIZE];
if (gz_is_available && filename && newfile) {
if (filename[0] && newfile[0]) {
// not: NOT an UTF-8 filename
gzFile gz = gzopen(filename, "rb");
if (gz) {
FILE*const fpout = FOPEN(fconv(catbuff, newfile), "wb");
int size=0;
FILE *const fpout = FOPEN(fconv(catbuff, newfile), "wb");
int size = 0;
if (fpout) {
int nr;
do {
char BIGSTK buff[1024];
nr=gzread (gz, buff, 1024);
if (nr>0) {
size+=nr;
if (fwrite(buff,1,nr,fpout) != nr)
nr=size=-1;
nr = gzread(gz, buff, 1024);
if (nr > 0) {
size += nr;
if (fwrite(buff, 1, nr, fpout) != nr)
nr = size = -1;
}
} while(nr>0);
} while(nr > 0);
fclose(fpout);
} else
size=-1;
size = -1;
gzclose(gz);
ret = (int) size;
}
@@ -87,47 +89,41 @@ int hts_zunpack(char* filename,char* newfile) {
return ret;
}
int hts_extract_meta(const char* path) {
char catbuff[CATBUFF_SIZE];
unzFile zFile = unzOpen(fconcat(catbuff,path,"hts-cache/new.zip"));
zipFile zFileOut = zipOpen(fconcat(catbuff,path,"hts-cache/meta.zip"), 0);
int hts_extract_meta(const char *path) {
char catbuff[CATBUFF_SIZE];
unzFile zFile = unzOpen(fconcat(catbuff, path, "hts-cache/new.zip"));
zipFile zFileOut = zipOpen(fconcat(catbuff, path, "hts-cache/meta.zip"), 0);
if (zFile != NULL && zFileOut != NULL) {
if (unzGoToFirstFile(zFile) == Z_OK) {
zip_fileinfo fi;
unz_file_info ufi;
char BIGSTK filename[HTS_URLMAXSIZE * 4];
char BIGSTK comment[8192];
memset(comment, 0, sizeof(comment)); // for truncated reads
memset(comment, 0, sizeof(comment)); // for truncated reads
memset(&fi, 0, sizeof(fi));
memset(&ufi, 0, sizeof(ufi));
do {
do {
int readSizeHeader;
filename[0] = '\0';
comment[0] = '\0';
if (unzOpenCurrentFile(zFile) == Z_OK) {
if (
(readSizeHeader = unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0
&&
unzGetCurrentFileInfo(zFile, &ufi, filename, sizeof(filename) - 2, NULL, 0, NULL, 0) == Z_OK
)
{
if ((readSizeHeader =
unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0
&& unzGetCurrentFileInfo(zFile, &ufi, filename,
sizeof(filename) - 2, NULL, 0, NULL,
0) == Z_OK) {
comment[readSizeHeader] = '\0';
fi.dosDate = ufi.dosDate;
fi.internal_fa = ufi.internal_fa;
fi.external_fa = ufi.external_fa;
if (zipOpenNewFileInZip(zFileOut,
filename,
&fi,
NULL,
0,
NULL,
0,
NULL, /* comment */
Z_DEFLATED,
Z_DEFAULT_COMPRESSION) == Z_OK)
{
if (zipWriteInFileInZip(zFileOut, comment, (int) strlen(comment)) != Z_OK) {
if (zipOpenNewFileInZip(zFileOut, filename, &fi, NULL, 0, NULL, 0, NULL, /* comment */
Z_DEFLATED, Z_DEFAULT_COMPRESSION) == Z_OK) {
if (zipWriteInFileInZip(zFileOut, comment, (int) strlen(comment))
!= Z_OK) {
}
if (zipCloseFileInZip(zFileOut) != Z_OK) {
}
@@ -135,41 +131,41 @@ int hts_extract_meta(const char* path) {
}
unzCloseCurrentFile(zFile);
}
} while( unzGoToNextFile(zFile) == Z_OK );
} while(unzGoToNextFile(zFile) == Z_OK);
}
zipClose(zFileOut, "Meta-data extracted by HTTrack/"HTTRACK_VERSION);
zipClose(zFileOut, "Meta-data extracted by HTTrack/" HTTRACK_VERSION);
unzClose(zFile);
return 1;
}
return 0;
}
const char* hts_get_zerror(int err) {
switch(err) {
case UNZ_OK:
return "no error";
break;
case UNZ_END_OF_LIST_OF_FILE:
return "end of list of file";
break;
case UNZ_ERRNO:
return (const char*) strerror(errno);
break;
case UNZ_PARAMERROR:
return "parameter error";
break;
case UNZ_BADZIPFILE:
return "bad zip file";
break;
case UNZ_INTERNALERROR:
return "internal error";
break;
case UNZ_CRCERROR:
return "crc error";
break;
default:
return "unknown error";
break;
const char *hts_get_zerror(int err) {
switch (err) {
case UNZ_OK:
return "no error";
break;
case UNZ_END_OF_LIST_OF_FILE:
return "end of list of file";
break;
case UNZ_ERRNO:
return (const char *) strerror(errno);
break;
case UNZ_PARAMERROR:
return "parameter error";
break;
case UNZ_BADZIPFILE:
return "bad zip file";
break;
case UNZ_INTERNALERROR:
return "internal error";
break;
case UNZ_CRCERROR:
return "crc error";
break;
default:
return "unknown error";
break;
}
}
#endif

10
src/htszlib.h
View File

@@ -17,25 +17,21 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Unpacking subroutines using Jean-loup Gailly's Zlib */
/* for http compressed data */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTS_DEFZLIB
#define HTS_DEFZLIB
@@ -51,9 +47,9 @@ Please visit our Website: http://www.httrack.com
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
extern int gz_is_available;
extern int hts_zunpack(char* filename,char* newfile);
extern int hts_extract_meta(const char* path);
extern const char* hts_get_zerror(int err);
extern int hts_zunpack(char *filename, char *newfile);
extern int hts_extract_meta(const char *path);
extern const char *hts_get_zerror(int err);
#endif
#endif

198
src/httrack-library.h
View File

@@ -17,24 +17,20 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: HTTrack definition file for library usage */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTTRACK_DEFLIB
#define HTTRACK_DEFLIB
@@ -51,13 +47,13 @@ typedef struct strc_int2bytes2 strc_int2bytes2;
#ifndef HTS_DEF_DEFSTRUCT_hts_log_type
#define HTS_DEF_DEFSTRUCT_hts_log_type
typedef enum hts_log_type {
LOG_DEBUG,
LOG_INFO,
LOG_NOTICE,
LOG_WARNING,
LOG_ERROR,
LOG_PANIC,
LOG_ERRNO = 1 << 8
LOG_DEBUG,
LOG_INFO,
LOG_NOTICE,
LOG_WARNING,
LOG_ERROR,
LOG_PANIC,
LOG_ERRNO = 1 << 8
} hts_log_type;
#endif
@@ -98,104 +94,110 @@ HTSEXT_API void htsthread_wait(void);
/* Main functions */
HTSEXT_API int hts_main(int argc, char **argv);
HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt);
HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt);
/* Options handling */
HTSEXT_API httrackp* hts_create_opt(void);
HTSEXT_API void hts_free_opt(httrackp *opt);
HTSEXT_API void set_wrappers(httrackp *opt); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int plug_wrapper(httrackp *opt, const char *moduleName, const char* argv);
HTSEXT_API httrackp *hts_create_opt(void);
HTSEXT_API void hts_free_opt(httrackp * opt);
HTSEXT_API void set_wrappers(httrackp * opt); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int plug_wrapper(httrackp * opt, const char *moduleName,
const char *argv);
/* Logging */
HTSEXT_API int hts_log(httrackp *opt, const char* prefix, const char *msg);
HTSEXT_API void hts_log_print(httrackp *opt, int type, const char *format, ...);
HTSEXT_API int hts_log(httrackp * opt, const char *prefix, const char *msg);
HTSEXT_API void hts_log_print(httrackp * opt, int type, const char *format,
...);
/* Infos */
HTSEXT_API const char* hts_get_version_info(httrackp *opt);
HTSEXT_API const char* hts_is_available(void);
HTSEXT_API const char *hts_get_version_info(httrackp * opt);
HTSEXT_API const char *hts_is_available(void);
/* Wrapper functions */
HTSEXT_API int htswrap_init(void); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int htswrap_free(void); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int htswrap_add(httrackp *opt, const char* name, void* fct);
HTSEXT_API unsigned long int htswrap_read(httrackp *opt, const char* name);
HTSEXT_API int htswrap_set_userdef(httrackp *opt, void *userdef);
HTSEXT_API void* htswrap_get_userdef(httrackp *opt);
HTSEXT_API int htswrap_init(void); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int htswrap_free(void); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int htswrap_add(httrackp * opt, const char *name, void *fct);
HTSEXT_API unsigned long int htswrap_read(httrackp * opt, const char *name);
HTSEXT_API int htswrap_set_userdef(httrackp * opt, void *userdef);
HTSEXT_API void *htswrap_get_userdef(httrackp * opt);
/* Internal library allocators, if a different libc is being used by the client */
HTSEXT_API char* hts_strdup(const char* string);
HTSEXT_API void* hts_malloc(size_t size);
HTSEXT_API void* hts_realloc(void* data, size_t size);
HTSEXT_API void hts_free(void* data);
HTSEXT_API char *hts_strdup(const char *string);
HTSEXT_API void *hts_malloc(size_t size);
HTSEXT_API void *hts_realloc(void *data, size_t size);
HTSEXT_API void hts_free(void *data);
/* Other functions */
HTSEXT_API int hts_resetvar(void); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binpath);
HTSEXT_API const char* hts_getcategories(const char* path, int type);
HTSEXT_API const char* hts_getcategory(const char* filename);
HTSEXT_API int hts_resetvar(void); // DEPRECATED - DUMMY FUNCTION
HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
const char *binpath);
HTSEXT_API const char *hts_getcategories(const char *path, int type);
HTSEXT_API const char *hts_getcategory(const char *filename);
/* Catch-URL */
HTSEXT_API T_SOC catch_url_init_std(int* port_prox,char* adr_prox);
HTSEXT_API T_SOC catch_url_init(int* port,char* adr);
HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data);
HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox);
HTSEXT_API T_SOC catch_url_init(int *port, char *adr);
HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data);
/* State */
HTSEXT_API int hts_is_parsing(httrackp *opt, int flag);
HTSEXT_API int hts_is_testing(httrackp *opt);
HTSEXT_API int hts_is_exiting(httrackp *opt);
HTSEXT_API int hts_is_parsing(httrackp * opt, int flag);
HTSEXT_API int hts_is_testing(httrackp * opt);
HTSEXT_API int hts_is_exiting(httrackp * opt);
/*HTSEXT_API int hts_setopt(httrackp* opt); DEPRECATED ; see copy_htsopt() */
HTSEXT_API int hts_addurl(httrackp *opt, char** url);
HTSEXT_API int hts_resetaddurl(httrackp *opt);
HTSEXT_API int copy_htsopt(httrackp* from, httrackp* to);
HTSEXT_API char* hts_errmsg(httrackp *opt);
HTSEXT_API int hts_setpause(httrackp *opt, int); // pause transfer
HTSEXT_API int hts_request_stop(httrackp* opt, int force);
HTSEXT_API int hts_cancel_file_push(httrackp *opt, const char *url);
HTSEXT_API void hts_cancel_test(httrackp *opt);
HTSEXT_API void hts_cancel_parsing(httrackp *opt);
HTSEXT_API void hts_cancel_test(httrackp *opt);
HTSEXT_API void hts_cancel_parsing(httrackp *opt);
HTSEXT_API int hts_addurl(httrackp * opt, char **url);
HTSEXT_API int hts_resetaddurl(httrackp * opt);
HTSEXT_API int copy_htsopt(httrackp * from, httrackp * to);
HTSEXT_API char *hts_errmsg(httrackp * opt);
HTSEXT_API int hts_setpause(httrackp * opt, int); // pause transfer
HTSEXT_API int hts_request_stop(httrackp * opt, int force);
HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url);
HTSEXT_API void hts_cancel_test(httrackp * opt);
HTSEXT_API void hts_cancel_parsing(httrackp * opt);
HTSEXT_API void hts_cancel_test(httrackp * opt);
HTSEXT_API void hts_cancel_parsing(httrackp * opt);
/* Tools */
HTSEXT_API int structcheck(const char* path);
HTSEXT_API int dir_exists(const char* path);
HTSEXT_API void infostatuscode(char* msg,int statuscode);
HTSEXT_API int structcheck(const char *path);
HTSEXT_API int dir_exists(const char *path);
HTSEXT_API void infostatuscode(char *msg, int statuscode);
HTSEXT_API HTS_INLINE TStamp mtime_local(void);
HTSEXT_API void qsec2str(char *st,TStamp t);
HTSEXT_API char* int2char(strc_int2bytes2* strc, int n);
HTSEXT_API char* int2bytes(strc_int2bytes2* strc, LLint n);
HTSEXT_API char* int2bytessec(strc_int2bytes2* strc, long int n);
HTSEXT_API char** int2bytes2(strc_int2bytes2* strc, LLint n);
HTSEXT_API char* jump_identification(char*);
HTSEXT_API char* jump_normalized(char*);
HTSEXT_API char* jump_toport(char*);
HTSEXT_API char* fil_normalized(char* source, char* dest);
HTSEXT_API char* adr_normalized(char* source, char* dest);
HTSEXT_API char* hts_rootdir(char* file);
HTSEXT_API void qsec2str(char *st, TStamp t);
HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n);
HTSEXT_API char *int2bytes(strc_int2bytes2 * strc, LLint n);
HTSEXT_API char *int2bytessec(strc_int2bytes2 * strc, long int n);
HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n);
HTSEXT_API char *jump_identification(char *);
HTSEXT_API char *jump_normalized(char *);
HTSEXT_API char *jump_toport(char *);
HTSEXT_API char *fil_normalized(char *source, char *dest);
HTSEXT_API char *adr_normalized(char *source, char *dest);
HTSEXT_API char *hts_rootdir(char *file);
/* Escaping URLs */
HTSEXT_API void unescape_amp(char* s);
HTSEXT_API void escape_spc_url(char* s);
HTSEXT_API void escape_in_url(char* s);
HTSEXT_API void escape_uri(char* s);
HTSEXT_API void escape_uri_utf(char* s);
HTSEXT_API void escape_check_url(char* s);
HTSEXT_API char* escape_check_url_addr(char* s);
HTSEXT_API void x_escape_http(char* s,int mode);
HTSEXT_API char* unescape_http(char *catbuff, const char* s);
HTSEXT_API char* unescape_http_unharm(char *catbuff, const char* s, int no_high);
HTSEXT_API char* antislash_unescaped(char *catbuff, const char* s);
HTSEXT_API void escape_remove_control(char* s);
HTSEXT_API void get_httptype(httrackp *opt,char *s,const char *fil,int flag);
HTSEXT_API int is_knowntype(httrackp *opt,const char *fil);
HTSEXT_API int is_userknowntype(httrackp *opt,const char *fil);
HTSEXT_API void unescape_amp(char *s);
HTSEXT_API void escape_spc_url(char *s);
HTSEXT_API void escape_in_url(char *s);
HTSEXT_API void escape_uri(char *s);
HTSEXT_API void escape_uri_utf(char *s);
HTSEXT_API void escape_check_url(char *s);
HTSEXT_API char *escape_check_url_addr(char *s);
HTSEXT_API void x_escape_http(char *s, int mode);
HTSEXT_API char *unescape_http(char *catbuff, const char *s);
HTSEXT_API char *unescape_http_unharm(char *catbuff, const char *s,
int no_high);
HTSEXT_API char *antislash_unescaped(char *catbuff, const char *s);
HTSEXT_API void escape_remove_control(char *s);
HTSEXT_API void get_httptype(httrackp * opt, char *s, const char *fil,
int flag);
HTSEXT_API int is_knowntype(httrackp * opt, const char *fil);
HTSEXT_API int is_userknowntype(httrackp * opt, const char *fil);
HTSEXT_API int is_dyntype(const char *fil);
HTSEXT_API char* get_ext(char *catbuff, const char *fil);
HTSEXT_API char *get_ext(char *catbuff, const char *fil);
/* Ugly string tools */
HTSEXT_API char* concat(char *catbuff,const char* a,const char* b) ;
HTSEXT_API char* fconcat(char *catbuff, const char* a, const char* b);
HTSEXT_API char* fconv(char *catbuff, const char* a);
HTSEXT_API char *concat(char *catbuff, const char *a, const char *b);
HTSEXT_API char *fconcat(char *catbuff, const char *a, const char *b);
HTSEXT_API char *fconv(char *catbuff, const char *a);
/* Debugging */
HTSEXT_API void hts_debug(int level);
@@ -205,7 +207,7 @@ HTSEXT_API void hts_debug(int level);
#ifndef HTS_DEF_FWSTRUCT_find_handle_struct
#define HTS_DEF_FWSTRUCT_find_handle_struct
typedef struct find_handle_struct find_handle_struct;
typedef find_handle_struct* find_handle;
typedef find_handle_struct *find_handle;
#endif
#ifndef HTS_DEF_FWSTRUCT_topindex_chain
@@ -213,15 +215,15 @@ typedef find_handle_struct* find_handle;
typedef struct topindex_chain topindex_chain;
#endif
struct topindex_chain {
int level; /* sort level */
char* category; /* category */
char name[2048]; /* path */
struct topindex_chain* next; /* next element */
int level; /* sort level */
char *category; /* category */
char name[2048]; /* path */
struct topindex_chain *next; /* next element */
};
HTSEXT_API find_handle hts_findfirst(char* path);
HTSEXT_API find_handle hts_findfirst(char *path);
HTSEXT_API int hts_findnext(find_handle find);
HTSEXT_API int hts_findclose(find_handle find);
HTSEXT_API char* hts_findgetname(find_handle find);
HTSEXT_API char *hts_findgetname(find_handle find);
HTSEXT_API int hts_findgetsize(find_handle find);
HTSEXT_API int hts_findisdir(find_handle find);
HTSEXT_API int hts_findisfile(find_handle find);
@@ -231,27 +233,35 @@ HTSEXT_API int hts_findissystem(find_handle find);
#ifndef HTS_DEF_FILEAPI
#ifdef _WIN32
#define FOPEN hts_fopen_utf8
HTSEXT_API FILE* hts_fopen_utf8(const char *path, const char *mode);
HTSEXT_API FILE *hts_fopen_utf8(const char *path, const char *mode);
#define STAT hts_stat_utf8
typedef struct _stat STRUCT_STAT;
HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT *buf);
HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT * buf);
#define UNLINK hts_unlink_utf8
HTSEXT_API int hts_unlink_utf8(const char *pathname);
#define RENAME hts_rename_utf8
HTSEXT_API int hts_rename_utf8(const char *oldpath, const char *newpath);
#define MKDIR(F) hts_mkdir_utf8(F)
HTSEXT_API int hts_mkdir_utf8(const char *pathname);
#define UTIME(A,B) hts_utime_utf8(A,B)
typedef struct _utimbuf STRUCT_UTIMBUF;
HTSEXT_API int hts_utime_utf8(const char *filename, const STRUCT_UTIMBUF *times);
HTSEXT_API int hts_utime_utf8(const char *filename,
const STRUCT_UTIMBUF * times);
#else
#define FOPEN fopen
#define STAT stat
typedef struct stat STRUCT_STAT;
#define UNLINK unlink
#define RENAME rename
#define MKDIR(F) mkdir(F, HTS_ACCESS_FOLDER)
typedef struct utimbuf STRUCT_UTIMBUF;
#define UTIME(A,B) utime(A,B)
#endif
#define HTS_DEF_FILEAPI

841
src/httrack.c
View File

File diff suppressed because it is too large Load Diff

21
src/httrack.h
View File

@@ -17,25 +17,22 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: htsshow.c console progress info */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTSTOOLS_DEFH
#define HTSTOOLS_DEFH
#define HTSTOOLS_DEFH
#include "htsglobal.h"
#include "htscore.h"
@@ -48,16 +45,16 @@ struct t_StatsBuffer {
char name[1024];
char file[1024];
char state[256];
char BIGSTK url_sav[HTS_URLMAXSIZE*2]; // pour cancel
char BIGSTK url_adr[HTS_URLMAXSIZE*2];
char BIGSTK url_fil[HTS_URLMAXSIZE*2];
char BIGSTK url_sav[HTS_URLMAXSIZE * 2]; // pour cancel
char BIGSTK url_adr[HTS_URLMAXSIZE * 2];
char BIGSTK url_fil[HTS_URLMAXSIZE * 2];
LLint size;
LLint sizetot;
int offset;
//
int back;
//
int actived; // pour disabled
int actived; // pour disabled
};
#ifndef HTS_DEF_FWSTRUCT_t_InpInfo
@@ -90,13 +87,14 @@ int main(int argc, char **argv);
/* */
// Engine internal variables
typedef void (* htsErrorCallback)(char* msg, char* file, int line);
typedef void (*htsErrorCallback) (char *msg, char *file, int line);
extern HTSEXT_API htsErrorCallback htsCallbackErr;
extern HTSEXT_API int htsMemoryFastXfr;
/* */
extern HTSEXT_API hts_stat_struct HTS_STAT;
extern int _DEBUG_HEAD;
extern FILE* ioinfo;
extern FILE *ioinfo;
// from htsbase.h
@@ -187,8 +185,9 @@ extern FILE* ioinfo;
} while(0)
// emergency log
typedef void (*t_abortLog)(char* msg, char* file, int line);
typedef void (*t_abortLog) (char *msg, char *file, int line);
extern HTSEXT_API t_abortLog abortLog__;
#define abortLog(a) abortLog__(a, __FILE__, __LINE__)
#define abortLogFmt(a) do { \
FILE* fp = fopen("CRASH.TXT", "wb"); \

102
src/md5.c
View File

@@ -17,7 +17,7 @@
/* #include "config.h" */
#include <string.h> /* for memcpy() */
#include <string.h> /* for memcpy() */
#include "md5.h"
static void byteReverse(unsigned char *buf, unsigned longs);
@@ -30,69 +30,66 @@ static void byteReverse(unsigned char *buf, unsigned longs);
b ^= a; \
a ^= b; \
} while(0)
static void byteReverse(unsigned char *buf, unsigned longs)
{
/*uint32 t;*/
static void byteReverse(unsigned char *buf, unsigned longs) {
/*uint32 t; */
do {
/*
t = (uint32) ((unsigned) buf[3] << 8 | buf[2]) << 16 |
((unsigned) buf[1] << 8 | buf[0]);
*(uint32 *) buf = t;
*/
t = (uint32) ((unsigned) buf[3] << 8 | buf[2]) << 16 |
((unsigned) buf[1] << 8 | buf[0]);
*(uint32 *) buf = t;
*/
byteSwap(buf[0], buf[3]);
byteSwap(buf[1], buf[2]);
buf += 4;
} while (--longs);
} while(--longs);
}
/*
* Start MD5 accumulation. Set bit count to 0 and buffer to mysterious
* initialization constants.
*/
void MD5Init(struct MD5Context *ctx, int brokenEndian)
{
void MD5Init(struct MD5Context *ctx, int brokenEndian) {
ctx->buf[0] = 0x67452301;
ctx->buf[1] = 0xefcdab89;
ctx->buf[2] = 0x98badcfe;
ctx->buf[3] = 0x10325476;
ctx->bits[0] = 0;
ctx->bits[1] = 0;
/*#ifdef WORDS_BIGENDIAN*/
/*#ifdef WORDS_BIGENDIAN */
if (brokenEndian) {
ctx->doByteReverse = 0;
} else {
ctx->doByteReverse = 1;
}
/*#else
ctx->doByteReverse = 0;
#endif
*/
ctx->doByteReverse = 0;
#endif
*/
}
/*
* Update context to reflect the concatenation of another buffer full
* of bytes.
*/
void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len)
{
void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len) {
uint32 t;
/* Update bitcount */
t = ctx->bits[0];
if ((ctx->bits[0] = t + ((uint32) len << 3)) < t)
ctx->bits[1]++; /* Carry from low to high */
ctx->bits[1]++; /* Carry from low to high */
ctx->bits[1] += len >> 29;
t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */
t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */
/* Handle any leading odd-sized chunks */
if (t) {
unsigned char *p = (unsigned char *) ctx->in + t;
t = 64 - t;
if (len < t) {
memcpy(p, buf, len);
@@ -106,8 +103,8 @@ void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len)
len -= t;
}
/* Process data in 64-byte chunks */
while (len >= 64) {
while(len >= 64) {
memcpy(ctx->in, buf, 64);
if (ctx->doByteReverse)
byteReverse(ctx->in, 16);
@@ -115,9 +112,9 @@ void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len)
buf += 64;
len -= 64;
}
/* Handle any remaining bytes of data. */
memcpy(ctx->in, buf, len);
}
@@ -125,22 +122,21 @@ void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len)
* Final wrapup - pad to 64-byte boundary with the bit pattern
* 1 0* (64-bit count of bits processed, MSB-first)
*/
void MD5Final(unsigned char digest[16], struct MD5Context *ctx)
{
void MD5Final(unsigned char digest[16], struct MD5Context *ctx) {
unsigned count;
unsigned char *p;
/* Compute number of bytes mod 64 */
count = (ctx->bits[0] >> 3) & 0x3F;
/* Set the first char of padding to 0x80. This is safe since there is
always at least one byte free */
always at least one byte free */
p = ctx->in + count;
*p++ = 0x80;
/* Bytes of padding needed to make 64 bytes */
count = 64 - 1 - count;
/* Pad out to 56 mod 64 */
if (count < 8) {
/* Two lots of padding: Pad the first block to 64 bytes */
@@ -148,7 +144,7 @@ void MD5Final(unsigned char digest[16], struct MD5Context *ctx)
if (ctx->doByteReverse)
byteReverse(ctx->in, 16);
MD5Transform(ctx->buf, (uint32 *) ctx->in);
/* Now fill the next block with 56 bytes */
memset(ctx->in, 0, 56);
} else {
@@ -157,19 +153,19 @@ void MD5Final(unsigned char digest[16], struct MD5Context *ctx)
}
if (ctx->doByteReverse)
byteReverse(ctx->in, 14);
/* Append length in bits and transform */
/* Note: see patch for PAM from Tomas Mraz */
memcpy((uint32 *)ctx->in + 14, ctx->bits, 2*sizeof(uint32));
memcpy((uint32 *) ctx->in + 14, ctx->bits, 2 * sizeof(uint32));
/*((uint32 *) ctx->in)[14] = ctx->bits[0];
((uint32 *) ctx->in)[15] = ctx->bits[1];
*/
((uint32 *) ctx->in)[15] = ctx->bits[1];
*/
MD5Transform(ctx->buf, (uint32 *) ctx->in);
if (ctx->doByteReverse)
byteReverse((unsigned char *) ctx->buf, 4);
memcpy(digest, ctx->buf, 16);
memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */
memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */
}
/* The four core functions - F1 is optimized somewhat */
@@ -189,15 +185,14 @@ void MD5Final(unsigned char digest[16], struct MD5Context *ctx)
* reflect the addition of 16 longwords of new data. MD5Update blocks
* the data and converts bytes into longwords for this routine.
*/
void MD5Transform(uint32 buf[4], uint32 const in[16])
{
void MD5Transform(uint32 buf[4], uint32 const in[16]) {
register uint32 a, b, c, d;
a = buf[0];
b = buf[1];
c = buf[2];
d = buf[3];
MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);
MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);
MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);
@@ -214,7 +209,7 @@ void MD5Transform(uint32 buf[4], uint32 const in[16])
MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);
MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);
MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);
MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
@@ -231,7 +226,7 @@ void MD5Transform(uint32 buf[4], uint32 const in[16])
MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);
MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);
MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);
MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);
MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);
MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
@@ -248,7 +243,7 @@ void MD5Transform(uint32 buf[4], uint32 const in[16])
MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);
MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);
MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);
MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
@@ -265,10 +260,9 @@ void MD5Transform(uint32 buf[4], uint32 const in[16])
MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);
MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);
buf[0] += a;
buf[1] += b;
buf[2] += c;
buf[3] += d;
}

215
src/proxy/main.c
View File

@@ -40,45 +40,47 @@ Please visit our Website: http://www.httrack.com
#ifndef _WIN32
#include <signal.h>
static void sig_brpipe( int code ) {
static void sig_brpipe(int code) {
/* ignore */
}
#endif
static int scanHostPort(const char* str, char *host, int *port) {
char* pos = strrchr(str, ':');
if (pos != NULL) {
int n = (int) ( pos - str );
if (n < 256) {
host[0] = '\0';
strncat(host, str, n);
if (sscanf(pos + 1, "%d", port) == 1) {
return 1;
}
}
}
return 0;
static int scanHostPort(const char *str, char *host, int *port) {
char *pos = strrchr(str, ':');
if (pos != NULL) {
int n = (int) (pos - str);
if (n < 256) {
host[0] = '\0';
strncat(host, str, n);
if (sscanf(pos + 1, "%d", port) == 1) {
return 1;
}
}
}
return 0;
}
int main(int argc, char* argv[])
{
int main(int argc, char *argv[]) {
int i;
int ret = 0;
int proxyPort = 0, icpPort = 0;
char proxyAddr[256 + 1], icpAddr[256 + 1];
PT_Indexes index;
int proxyPort = 0, icpPort = 0;
char proxyAddr[256 + 1], icpAddr[256 + 1];
PT_Indexes index;
#ifdef _WIN32
{
WORD wVersionRequested; // requested version WinSock API
WORD wVersionRequested; // requested version WinSock API
WSADATA wsadata; // Windows Sockets API data
int stat;
wVersionRequested = 0x0101;
stat = WSAStartup( wVersionRequested, &wsadata );
stat = WSAStartup(wVersionRequested, &wsadata);
if (stat != 0) {
fprintf(stderr, "Winsock not found!\n");
return -1;
} else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) {
} else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) {
fprintf(stderr, "WINSOCK.DLL does not support version 1.1\n");
WSACleanup();
return -1;
@@ -86,93 +88,103 @@ int main(int argc, char* argv[])
}
#endif
/* Args */
printf("ProxyTrack %s, build proxies upon HTTrack Website Copier Archives\n", PROXYTRACK_VERSION);
printf("Copyright (C) Xavier Roche and other contributors\n");
printf("\n");
printf("This program is free software; you can redistribute it and/or\n");
printf("modify it under the terms of the GNU General Public License\n");
printf("as published by the Free Software Foundation; either version 3\n");
printf("of the License, or any later version.\n");
printf("\n");
printf("*** This version is a development release ***\n");
printf("\n");
if (argc < 3
|| (
strcmp(argv[1], "--convert") != 0
&&
(
!scanHostPort(argv[1], proxyAddr, &proxyPort)
|| !scanHostPort(argv[2], icpAddr, &icpPort)
)
)
)
{
fprintf(stderr, "proxy mode:\n");
fprintf(stderr, "usage: %s <proxy-addr:proxy-port> <ICP-addr:ICP-port> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n", argv[0]);
fprintf(stderr, "\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n", argv[0]);
fprintf(stderr, "convert mode:\n");
fprintf(stderr, "usage: %s --convert <archive-output-path> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n", argv[0]);
fprintf(stderr, "\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n", argv[0]);
/* Args */
printf("ProxyTrack %s, build proxies upon HTTrack Website Copier Archives\n",
PROXYTRACK_VERSION);
printf("Copyright (C) Xavier Roche and other contributors\n");
printf("\n");
printf("This program is free software; you can redistribute it and/or\n");
printf("modify it under the terms of the GNU General Public License\n");
printf("as published by the Free Software Foundation; either version 3\n");
printf("of the License, or any later version.\n");
printf("\n");
printf("*** This version is a development release ***\n");
printf("\n");
if (argc < 3
|| (strcmp(argv[1], "--convert") != 0
&& (!scanHostPort(argv[1], proxyAddr, &proxyPort)
|| !scanHostPort(argv[2], icpAddr, &icpPort)
)
)
) {
fprintf(stderr, "proxy mode:\n");
fprintf(stderr,
"usage: %s <proxy-addr:proxy-port> <ICP-addr:ICP-port> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n",
argv[0]);
fprintf(stderr,
"\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n",
argv[0]);
fprintf(stderr, "convert mode:\n");
fprintf(stderr,
"usage: %s --convert <archive-output-path> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n",
argv[0]);
fprintf(stderr,
"\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n",
argv[0]);
return 1;
}
index = PT_New();
for(i = 3 ; i < argc ; i++) {
if (argv[i][0] == '-') {
if (strcmp(argv[i], "--list") == 0) {
if (i + 1 < argc) {
char line[256 + 1];
FILE *fp = fopen(argv[++i], "rb");
if (fp == NULL) {
fprintf(stderr, "error: could not process list %s\n", argv[i]);
exit(1);
}
while(linput(fp, line, 256)) {
int itemsAdded = PT_AddIndex(index, line);
if (itemsAdded > 0) {
fprintf(stderr, "processed: %s (%d items added)\n", line, itemsAdded);
} else if (itemsAdded == 0) {
fprintf(stderr, "processed: %s (no items added)\n", line);
} else {
fprintf(stderr, "error: could not process %s\n", line);
}
}
fclose(fp);
}
} else {
fprintf(stderr, "* bad arg %s\n", argv[i]);
exit(1);
}
} else {
int itemsAdded = PT_AddIndex(index, argv[i]);
if (itemsAdded > 0) {
fprintf(stderr, "processed: %s (%d items added)\n", argv[i], itemsAdded);
} else if (itemsAdded == 0) {
fprintf(stderr, "processed: %s (no items added)\n", argv[i]);
} else {
fprintf(stderr, "error: could not process %s\n", argv[i]);
}
}
}
index = PT_New();
for(i = 3; i < argc; i++) {
if (argv[i][0] == '-') {
if (strcmp(argv[i], "--list") == 0) {
if (i + 1 < argc) {
char line[256 + 1];
FILE *fp = fopen(argv[++i], "rb");
/* sigpipe */
if (fp == NULL) {
fprintf(stderr, "error: could not process list %s\n", argv[i]);
exit(1);
}
while(linput(fp, line, 256)) {
int itemsAdded = PT_AddIndex(index, line);
if (itemsAdded > 0) {
fprintf(stderr, "processed: %s (%d items added)\n", line,
itemsAdded);
} else if (itemsAdded == 0) {
fprintf(stderr, "processed: %s (no items added)\n", line);
} else {
fprintf(stderr, "error: could not process %s\n", line);
}
}
fclose(fp);
}
} else {
fprintf(stderr, "* bad arg %s\n", argv[i]);
exit(1);
}
} else {
int itemsAdded = PT_AddIndex(index, argv[i]);
if (itemsAdded > 0) {
fprintf(stderr, "processed: %s (%d items added)\n", argv[i],
itemsAdded);
} else if (itemsAdded == 0) {
fprintf(stderr, "processed: %s (no items added)\n", argv[i]);
} else {
fprintf(stderr, "error: could not process %s\n", argv[i]);
}
}
}
/* sigpipe */
#ifndef _WIN32
signal( SIGPIPE , sig_brpipe ); // broken pipe (write into non-opened socket)
signal(SIGPIPE, sig_brpipe); // broken pipe (write into non-opened socket)
#endif
/* Go */
if (strcmp(argv[1], "--convert") != 0) {
ret = proxytrack_main(proxyAddr, proxyPort, icpAddr, icpPort, index);
} else {
if ((ret = PT_SaveCache(index, argv[2])) == 0) {
fprintf(stderr, "processed: '%s'\n", argv[2]);
} else {
fprintf(stderr, "error: could not save '%s'\n", argv[2]);
}
}
if (strcmp(argv[1], "--convert") != 0) {
ret = proxytrack_main(proxyAddr, proxyPort, icpAddr, icpPort, index);
} else {
if ((ret = PT_SaveCache(index, argv[2])) == 0) {
fprintf(stderr, "processed: '%s'\n", argv[2]);
} else {
fprintf(stderr, "error: could not save '%s'\n", argv[2]);
}
}
/* Wipe */
PT_Delete(index);
/* Wipe */
PT_Delete(index);
#ifdef _WIN32
WSACleanup();
@@ -180,4 +192,3 @@ int main(int argc, char* argv[])
return ret;
}

81
src/proxy/proxystrings.h
View File

@@ -20,7 +20,6 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Strings */
/* Author: Xavier Roche */
@@ -29,89 +28,93 @@ Please visit our Website: http://www.httrack.com
// Strings a bit safer than static buffers
#ifndef HTS_PROXYSTRINGS_DEFSTATIC
#define HTS_PROXYSTRINGS_DEFSTATIC
#define HTS_PROXYSTRINGS_DEFSTATIC
#include "htsstrings.h"
/* Tools */
static int ehexh(char c) {
if ((c>='0') && (c<='9')) return c-'0';
if ((c>='a') && (c<='f')) c-=('a'-'A');
if ((c>='A') && (c<='F')) return (c-'A'+10);
if ((c >= '0') && (c <= '9'))
return c - '0';
if ((c >= 'a') && (c <= 'f'))
c -= ('a' - 'A');
if ((c >= 'A') && (c <= 'F'))
return (c - 'A' + 10);
return 0;
}
static int ehex(const char* s) {
return 16*ehexh(*s)+ehexh(*(s+1));
static int ehex(const char *s) {
return 16 * ehexh(*s) + ehexh(*(s + 1));
}
static void unescapehttp(const char* s, String* tempo) {
static void unescapehttp(const char *s, String * tempo) {
int i;
for (i = 0; s[i] != '\0' ; i++) {
if (s[i]=='%' && s[i+1]=='%') {
for(i = 0; s[i] != '\0'; i++) {
if (s[i] == '%' && s[i + 1] == '%') {
i++;
StringAddchar(*tempo, '%');
} else if (s[i]=='%') {
} else if (s[i] == '%') {
char hc;
i++;
hc = (char) ehex(s+i);
hc = (char) ehex(s + i);
StringAddchar(*tempo, (char) hc);
i++; // sauter 2 caractères finalement
}
else if (s[i]=='+') {
i++; // sauter 2 caractères finalement
} else if (s[i] == '+') {
StringAddchar(*tempo, ' ');
}
else
} else
StringAddchar(*tempo, s[i]);
}
}
static void escapexml(const char* s, String* tempo) {
static void escapexml(const char *s, String * tempo) {
int i;
for (i=0 ; s[i] != '\0' ; i++) {
for(i = 0; s[i] != '\0'; i++) {
if (s[i] == '&')
StringCat(*tempo, "&amp;");
else if (s[i] == '<')
else if (s[i] == '<')
StringCat(*tempo, "&lt;");
else if (s[i] == '>')
else if (s[i] == '>')
StringCat(*tempo, "&gt;");
else if (s[i] == '\"')
else if (s[i] == '\"')
StringCat(*tempo, "&quot;");
else
StringAddchar(*tempo, s[i]);
}
}
static char* concat(char *catbuff,const char* a,const char* b) {
if (a != NULL && a[0] != '\0') {
strcpy(catbuff, a);
} else {
catbuff[0] = '\0';
}
if (b != NULL && b[0] != '\0') {
strcat(catbuff, b);
}
static char *concat(char *catbuff, const char *a, const char *b) {
if (a != NULL && a[0] != '\0') {
strcpy(catbuff, a);
} else {
catbuff[0] = '\0';
}
if (b != NULL && b[0] != '\0') {
strcat(catbuff, b);
}
return catbuff;
}
static char* __fconv(char* a) {
static char *__fconv(char *a) {
#ifdef WIN32
int i;
for(i = 0 ; a[i] != 0 ; i++)
if (a[i] == '/') // Unix-to-DOS style
for(i = 0; a[i] != 0; i++)
if (a[i] == '/') // Unix-to-DOS style
a[i] = '\\';
#endif
return a;
}
static char* fconcat(char *catbuff, const char* a, const char* b) {
return __fconv(concat(catbuff,a,b));
static char *fconcat(char *catbuff, const char *a, const char *b) {
return __fconv(concat(catbuff, a, b));
}
static char* fconv(char *catbuff, const char* a) {
return __fconv(concat(catbuff,a,""));
static char *fconv(char *catbuff, const char *a) {
return __fconv(concat(catbuff, a, ""));
}
#endif

2464
src/proxy/proxytrack.c
View File

File diff suppressed because it is too large Load Diff

328
src/proxy/proxytrack.h
View File

@@ -53,9 +53,8 @@ Please visit our Website: http://www.httrack.com
/* generic */
int proxytrack_main(char* proxyAddr, int proxyPort,
char* icpAddr, int icpPort,
PT_Indexes index);
int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort,
PT_Indexes index);
/* Spaces: CR,LF,TAB,FF */
#define is_space(c) ( ((c)==' ') || ((c)=='\"') || ((c)==10) || ((c)==13) || ((c)==9) || ((c)==12) || ((c)==11) || ((c)=='\'') )
@@ -71,6 +70,7 @@ static void proxytrack_print_log(const char *severity, const char *format, ...)
const int error = errno;
FILE *const fp = stderr;
va_list args;
fprintf(fp, " * %s: ", severity);
va_start(args, format);
(void) vfprintf(fp, format, args);
@@ -110,90 +110,105 @@ static void proxytrack_print_log(const char *severity, const char *format, ...)
"<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \
"<!-- End Disable IE Friendly HTTP Error Messages -->\r\n"
static char* gethomedir(void) {
char* home = getenv( "HOME" );
static char *gethomedir(void) {
char *home = getenv("HOME");
if (home)
return home;
else
return ".";
}
static int linput(FILE* fp,char* s,int max) {
static int linput(FILE * fp, char *s, int max) {
int c;
int j=0;
int j = 0;
do {
c=fgetc(fp);
if (c!=EOF) {
switch(c) {
case 13: break; // sauter CR
case 10: c=-1; break;
case 0: case 9: case 12: break; // sauter ces caractères
default: s[j++]=(char) c; break;
c = fgetc(fp);
if (c != EOF) {
switch (c) {
case 13:
break; // sauter CR
case 10:
c = -1;
break;
case 0:
case 9:
case 12:
break; // sauter ces caractères
default:
s[j++] = (char) c;
break;
}
}
} while((c!=-1) && (c!=EOF) && (j<(max-1)));
s[j]='\0';
} while((c != -1) && (c != EOF) && (j < (max - 1)));
s[j] = '\0';
return j;
}
static int link_has_authority(const char* lien) {
const char* a=lien;
if (isalpha((const unsigned char)*a)) {
static int link_has_authority(const char *lien) {
const char *a = lien;
if (isalpha((const unsigned char) *a)) {
// Skip scheme?
while (isalpha((const unsigned char)*a))
while(isalpha((const unsigned char) *a))
a++;
if (*a == ':')
a++;
else
return 0;
}
if (strncmp(a,"//",2) == 0)
if (strncmp(a, "//", 2) == 0)
return 1;
return 0;
}
static const char* jump_protocol(const char* source) {
static const char *jump_protocol(const char *source) {
int p;
// scheme
// "Comparisons of scheme names MUST be case-insensitive" (RFC2616)
if ((p = strfield(source,"http:")))
source+=p;
else if ((p = strfield(source,"ftp:")))
source+=p;
else if ((p = strfield(source,"https:")))
source+=p;
else if ((p = strfield(source,"file:")))
source+=p;
else if ((p = strfield(source,"mms:")))
source+=p;
if ((p = strfield(source, "http:")))
source += p;
else if ((p = strfield(source, "ftp:")))
source += p;
else if ((p = strfield(source, "https:")))
source += p;
else if ((p = strfield(source, "file:")))
source += p;
else if ((p = strfield(source, "mms:")))
source += p;
// net_path
if (strncmp(source,"//",2)==0)
source+=2;
if (strncmp(source, "//", 2) == 0)
source += 2;
return source;
}
static const char* strrchr_limit(const char* s, char c, const char* limit) {
static const char *strrchr_limit(const char *s, char c, const char *limit) {
if (limit == NULL) {
char* p = strrchr(s, c);
return p?(p+1):NULL;
char *p = strrchr(s, c);
return p ? (p + 1) : NULL;
} else {
char *a=NULL, *p;
char *a = NULL, *p;
for(;;) {
p=strchr((a)?a:s, c);
p = strchr((a) ? a : s, c);
if ((p >= limit) || (p == NULL))
return a;
a=p+1;
a = p + 1;
}
}
}
static const char* jump_protocol_and_auth(const char* source) {
const char *a,*trytofind;
static const char *jump_protocol_and_auth(const char *source) {
const char *a, *trytofind;
if (strcmp(source, "file://") == 0)
return source;
return source;
a = jump_protocol(source);
trytofind = strrchr_limit(a, '@', strchr(a,'/'));
return (trytofind != NULL)?trytofind:a;
trytofind = strrchr_limit(a, '@', strchr(a, '/'));
return (trytofind != NULL) ? trytofind : a;
}
#ifndef min
@@ -202,27 +217,29 @@ static const char* jump_protocol_and_auth(const char* source) {
#ifndef max
#define max(a,b) ((a)>(b)?(a):(b))
#endif
static int linput_trim(FILE* fp,char* s,int max) {
int rlen=0;
char* ls=(char*) malloc(max+2);
s[0]='\0';
static int linput_trim(FILE * fp, char *s, int max) {
int rlen = 0;
char *ls = (char *) malloc(max + 2);
s[0] = '\0';
if (ls) {
char* a;
char *a;
// lire ligne
rlen=linput(fp,ls,max);
rlen = linput(fp, ls, max);
if (rlen) {
// sauter espaces et tabs en fin
while( (rlen>0) && is_realspace(ls[max(rlen-1,0)]) )
ls[--rlen]='\0';
while((rlen > 0) && is_realspace(ls[max(rlen - 1, 0)]))
ls[--rlen] = '\0';
// sauter espaces en début
a=ls;
while((rlen>0) && ((*a==' ') || (*a=='\t'))) {
a = ls;
while((rlen > 0) && ((*a == ' ') || (*a == '\t'))) {
a++;
rlen--;
}
if (rlen>0) {
memcpy(s,a,rlen); // can copy \0 chars
s[rlen]='\0';
if (rlen > 0) {
memcpy(s, a, rlen); // can copy \0 chars
s[rlen] = '\0';
}
}
//
@@ -234,8 +251,9 @@ static int linput_trim(FILE* fp,char* s,int max) {
#ifndef S_ISREG
#define S_ISREG(m) ((m) & _S_IFREG)
#endif
static int fexist(char* s) {
static int fexist(char *s) {
struct stat st;
memset(&st, 0, sizeof(st));
if (stat(s, &st) == 0) {
if (S_ISREG(st.st_mode)) {
@@ -243,137 +261,155 @@ static int fexist(char* s) {
}
}
return 0;
}
}
/* convertir une chaine en temps */
static void set_lowcase(char* s) {
static void set_lowcase(char *s) {
int i;
for(i=0;i<(int) strlen(s);i++)
if ((s[i]>='A') && (s[i]<='Z'))
s[i]+=('a'-'A');
for(i = 0; i < (int) strlen(s); i++)
if ((s[i] >= 'A') && (s[i] <= 'Z'))
s[i] += ('a' - 'A');
}
static struct tm* convert_time_rfc822(struct tm *result,const char* s) {
char months[]="jan feb mar apr may jun jul aug sep oct nov dec";
static struct tm *convert_time_rfc822(struct tm *result, const char *s) {
char months[] = "jan feb mar apr may jun jul aug sep oct nov dec";
char str[256];
char* a;
char *a;
/* */
int result_mm=-1;
int result_dd=-1;
int result_n1=-1;
int result_n2=-1;
int result_n3=-1;
int result_n4=-1;
int result_mm = -1;
int result_dd = -1;
int result_n1 = -1;
int result_n2 = -1;
int result_n3 = -1;
int result_n4 = -1;
/* */
if ((int) strlen(s) > 200)
return NULL;
strcpy(str,s);
strcpy(str, s);
set_lowcase(str);
/* éliminer :,- */
while( (a=strchr(str,'-')) ) *a=' ';
while( (a=strchr(str,':')) ) *a=' ';
while( (a=strchr(str,',')) ) *a=' ';
while((a = strchr(str, '-')))
*a = ' ';
while((a = strchr(str, ':')))
*a = ' ';
while((a = strchr(str, ',')))
*a = ' ';
/* tokeniser */
a=str;
a = str;
while(*a) {
char *first, *last;
char tok[256];
/* découper mot */
while(*a==' ') a++; /* sauter espaces */
first=a;
while((*a) && (*a!=' ')) a++;
last=a;
tok[0]='\0';
if (first!=last) {
char* pos;
strncat(tok,first,(int) (last - first));
while(*a == ' ')
a++; /* sauter espaces */
first = a;
while((*a) && (*a != ' '))
a++;
last = a;
tok[0] = '\0';
if (first != last) {
char *pos;
strncat(tok, first, (int) (last - first));
/* analyser */
if ( (pos=strstr(months,tok)) ) { /* month always in letters */
result_mm=((int) (pos - months))/4;
if ((pos = strstr(months, tok))) { /* month always in letters */
result_mm = ((int) (pos - months)) / 4;
} else {
int number;
if (sscanf(tok,"%d",&number) == 1) { /* number token */
if (result_dd<0) /* day always first number */
result_dd=number;
else if (result_n1<0)
result_n1=number;
else if (result_n2<0)
result_n2=number;
else if (result_n3<0)
result_n3=number;
else if (result_n4<0)
result_n4=number;
} /* sinon, bruit de fond(+1GMT for exampel) */
if (sscanf(tok, "%d", &number) == 1) { /* number token */
if (result_dd < 0) /* day always first number */
result_dd = number;
else if (result_n1 < 0)
result_n1 = number;
else if (result_n2 < 0)
result_n2 = number;
else if (result_n3 < 0)
result_n3 = number;
else if (result_n4 < 0)
result_n4 = number;
} /* sinon, bruit de fond(+1GMT for exampel) */
}
}
}
if ((result_n1>=0) && (result_mm>=0) && (result_dd>=0) && (result_n2>=0) && (result_n3>=0) && (result_n4>=0)) {
if (result_n4>=1000) { /* Sun Nov 6 08:49:37 1994 */
result->tm_year=result_n4-1900;
result->tm_hour=result_n1;
result->tm_min=result_n2;
result->tm_sec=max(result_n3,0);
} else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */
result->tm_hour=result_n2;
result->tm_min=result_n3;
result->tm_sec=max(result_n4,0);
if (result_n1<=50) /* 00 means 2000 */
result->tm_year=result_n1+100;
else if (result_n1<1000) /* 99 means 1999 */
result->tm_year=result_n1;
else /* 2000 */
result->tm_year=result_n1-1900;
if ((result_n1 >= 0) && (result_mm >= 0) && (result_dd >= 0)
&& (result_n2 >= 0) && (result_n3 >= 0) && (result_n4 >= 0)) {
if (result_n4 >= 1000) { /* Sun Nov 6 08:49:37 1994 */
result->tm_year = result_n4 - 1900;
result->tm_hour = result_n1;
result->tm_min = result_n2;
result->tm_sec = max(result_n3, 0);
} else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */
result->tm_hour = result_n2;
result->tm_min = result_n3;
result->tm_sec = max(result_n4, 0);
if (result_n1 <= 50) /* 00 means 2000 */
result->tm_year = result_n1 + 100;
else if (result_n1 < 1000) /* 99 means 1999 */
result->tm_year = result_n1;
else /* 2000 */
result->tm_year = result_n1 - 1900;
}
result->tm_isdst=0; /* assume GMT */
result->tm_yday=-1; /* don't know */
result->tm_wday=-1; /* don't know */
result->tm_mon=result_mm;
result->tm_mday=result_dd;
result->tm_isdst = 0; /* assume GMT */
result->tm_yday = -1; /* don't know */
result->tm_wday = -1; /* don't know */
result->tm_mon = result_mm;
result->tm_mday = result_dd;
return result;
}
return NULL;
}
static struct tm PT_GetTime(time_t t) {
struct tm tmbuf;
struct tm tmbuf;
#ifdef _WIN32
struct tm * tm = gmtime(&t);
struct tm *tm = gmtime(&t);
#else
struct tm * tm = gmtime_r(&t, &tmbuf);
struct tm *tm = gmtime_r(&t, &tmbuf);
#endif
if (tm != NULL)
return *tm;
else {
memset(&tmbuf, 0, sizeof(tmbuf));
return tmbuf;
}
if (tm != NULL)
return *tm;
else {
memset(&tmbuf, 0, sizeof(tmbuf));
return tmbuf;
}
}
static int set_filetime(const char* file, struct tm* tm_time) {
static int set_filetime(const char *file, struct tm *tm_time) {
struct utimbuf tim;
#ifndef HTS_DO_NOT_USE_FTIME
struct timeb B;
memset(&B, 0, sizeof(B));
B.timezone=0;
ftime( &B );
tim.actime = tim.modtime = mktime(tm_time) - B.timezone*60;
B.timezone = 0;
ftime(&B);
tim.actime = tim.modtime = mktime(tm_time) - B.timezone * 60;
#else
// bogus time (GMT/local)..
tim.actime=tim.modtime=mktime(tm_time);
tim.actime = tim.modtime = mktime(tm_time);
#endif
return utime(file, &tim);
}
static int set_filetime_time_t(const char* file, time_t t) {
if (t != (time_t) 0 && t != (time_t) -1) {
struct tm tm = PT_GetTime(t);
return set_filetime(file, &tm);
}
return -1;
static int set_filetime_time_t(const char *file, time_t t) {
if (t != (time_t) 0 && t != (time_t) - 1) {
struct tm tm = PT_GetTime(t);
return set_filetime(file, &tm);
}
return -1;
}
static int set_filetime_rfc822(const char* file, const char* date) {
struct tm buffer;
struct tm* tm_s = convert_time_rfc822(&buffer,date);
static int set_filetime_rfc822(const char *file, const char *date) {
struct tm buffer;
struct tm *tm_s = convert_time_rfc822(&buffer, date);
if (tm_s) {
return set_filetime(file,tm_s);
} else return -1;
return set_filetime(file, tm_s);
} else
return -1;
}
#endif

3408
src/proxy/store.c
View File

File diff suppressed because it is too large Load Diff

60
src/proxy/store.h
View File

@@ -50,66 +50,68 @@ typedef struct _PT_CacheItem _PT_CacheItem;
typedef struct _PT_CacheItem *PT_CacheItem;
typedef struct _PT_Element {
int indexId; // index identifier, if suitable (!= -1)
//
int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945)
char* adr; // adresse du bloc de mémoire, NULL=vide
char* headers; // adresse des en têtes si présents (RFC822 format)
size_t size; // taille fichier
char msg[1024]; // error message ("\0"=undefined)
char contenttype[64]; // content-type ("text/html" par exemple)
char charset[64]; // charset ("iso-8859-1" par exemple)
char* location; // on copie dedans éventuellement la véritable 'location'
char lastmodified[64]; // Last-Modified
char etag[64]; // Etag
char cdispo[256]; // Content-Disposition coupé
int indexId; // index identifier, if suitable (!= -1)
//
int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945)
char *adr; // adresse du bloc de mémoire, NULL=vide
char *headers; // adresse des en têtes si présents (RFC822 format)
size_t size; // taille fichier
char msg[1024]; // error message ("\0"=undefined)
char contenttype[64]; // content-type ("text/html" par exemple)
char charset[64]; // charset ("iso-8859-1" par exemple)
char *location; // on copie dedans éventuellement la véritable 'location'
char lastmodified[64]; // Last-Modified
char etag[64]; // Etag
char cdispo[256]; // Content-Disposition coupé
} _PT_Element;
typedef struct _PT_Element *PT_Element;
typedef enum PT_Fetch_Flags {
FETCH_HEADERS, // fetch headers
FETCH_BODY // fetch body
FETCH_HEADERS, // fetch headers
FETCH_BODY // fetch body
} PT_Fetch_Flags;
/* Locking */
#ifdef _WIN32
typedef void* PT_Mutex;
typedef void *PT_Mutex;
#else
typedef pthread_mutex_t PT_Mutex;
#endif
void MutexInit(PT_Mutex *pMutex);
void MutexLock(PT_Mutex *pMutex);
void MutexUnlock(PT_Mutex *pMutex);
void MutexFree(PT_Mutex *pMutex);
void MutexInit(PT_Mutex * pMutex);
void MutexLock(PT_Mutex * pMutex);
void MutexUnlock(PT_Mutex * pMutex);
void MutexFree(PT_Mutex * pMutex);
/* Indexes */
PT_Indexes PT_New(void);
void PT_Delete(PT_Indexes index);
PT_Element PT_ReadIndex(PT_Indexes indexes, const char* url, int flags);
int PT_LookupIndex(PT_Indexes indexes, const char* url);
PT_Element PT_ReadIndex(PT_Indexes indexes, const char *url, int flags);
int PT_LookupIndex(PT_Indexes indexes, const char *url);
int PT_AddIndex(PT_Indexes index, const char *path);
int PT_RemoveIndex(PT_Indexes index, int indexId);
int PT_IndexMerge(PT_Indexes indexes, PT_Index *pindex);
int PT_IndexMerge(PT_Indexes indexes, PT_Index * pindex);
PT_Index PT_GetIndex(PT_Indexes indexes, int indexId);
time_t PT_GetTimeIndex(PT_Indexes indexes);
/* Indexes list */
PT_Element PT_Index_HTML_BuildRootInfo(PT_Indexes indexes);
char ** PT_Enumerate(PT_Indexes indexes, const char *url, int subtree);
char **PT_Enumerate(PT_Indexes indexes, const char *url, int subtree);
void PT_Enumerate_Delete(char ***plist);
int PT_EnumCache(PT_Indexes indexes, int (*callback)(void *, const char *url, PT_Element), void *arg);
int PT_EnumCache(PT_Indexes indexes,
int (*callback) (void *, const char *url, PT_Element),
void *arg);
int PT_SaveCache(PT_Indexes indexes, const char *filename);
/* Index */
PT_Index PT_LoadCache(const char *filename);
void PT_Index_Delete(PT_Index *pindex);
PT_Element PT_ReadCache(PT_Index index, const char* url, int flags);
int PT_LookupCache(PT_Index index, const char* url);
void PT_Index_Delete(PT_Index * pindex);
PT_Element PT_ReadCache(PT_Index index, const char *url, int flags);
int PT_LookupCache(PT_Index index, const char *url);
time_t PT_Index_Timestamp(PT_Index index);
/* Elements*/
PT_Element PT_ElementNew(void);
void PT_Element_Delete(PT_Element *pentry);
void PT_Element_Delete(PT_Element * pentry);
#endif