mirror of
https://github.com/xroche/httrack.git
synced 2026-07-11 11:26:31 +03:00
Compare commits
1 Commits
master
...
fast-cache
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
caf210b295 |
169
src/htsback.c
169
src/htsback.c
@@ -1646,138 +1646,104 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// tester cache
|
// tester cache
|
||||||
if ((strcmp(adr, "file://")) /* pas fichier */
|
if ((strcmp(adr, "file://")) /* pas fichier */
|
||||||
&&((!test) || (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
|
&& ((!test) ||
|
||||||
&&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
|
(cache->type == 1)) /* cache prioritaire, laisser passer en test! */
|
||||||
#if HTS_FAST_CACHE
|
&&
|
||||||
|
((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
|
||||||
|
0))) { // si en test on ne doit pas utiliser le
|
||||||
|
// cache sinon telescopage avec le 302..
|
||||||
intptr_t hash_pos;
|
intptr_t hash_pos;
|
||||||
int hash_pos_return = 0;
|
int hash_pos_return = 0;
|
||||||
#else
|
|
||||||
char *a = NULL;
|
|
||||||
#endif
|
|
||||||
#if HTS_FAST_CACHE
|
|
||||||
if (cache->hashtable) {
|
if (cache->hashtable) {
|
||||||
#else
|
|
||||||
if (cache->use) {
|
|
||||||
#endif
|
|
||||||
char BIGSTK buff[HTS_URLMAXSIZE * 4];
|
char BIGSTK buff[HTS_URLMAXSIZE * 4];
|
||||||
|
|
||||||
#if HTS_FAST_CACHE
|
|
||||||
strcpybuff(buff, adr);
|
strcpybuff(buff, adr);
|
||||||
strcatbuff(buff, fil);
|
strcatbuff(buff, fil);
|
||||||
hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
|
hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
|
||||||
#else
|
|
||||||
buff[0] = '\0';
|
|
||||||
strcatbuff(buff, "\n");
|
|
||||||
strcatbuff(buff, adr);
|
|
||||||
strcatbuff(buff, "\n");
|
|
||||||
strcatbuff(buff, fil);
|
|
||||||
strcatbuff(buff, "\n");
|
|
||||||
a = strstr(cache->use, buff);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
|
|
||||||
#if HTS_FAST_CACHE
|
|
||||||
// negative values when data is not in cache
|
// negative values when data is not in cache
|
||||||
if (hash_pos_return < 0) {
|
if (hash_pos_return < 0) {
|
||||||
#else
|
if (!test) { // not test mode
|
||||||
if (a) {
|
/* note: no check with IS_DELAYED_EXT() enabled - postcheck by
|
||||||
#endif
|
* client please! */
|
||||||
if (!test) { // non mode test
|
if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
|
||||||
#if HTS_FAST_CACHE==0
|
fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
|
||||||
int pos = -1;
|
0) { // final file missing or empty
|
||||||
|
int found = 0;
|
||||||
|
|
||||||
a += strlen(buff);
|
/* It is possible that the file has been moved due to changes in
|
||||||
sscanf(a, "%d", &pos); // lire position
|
* build structure */
|
||||||
#endif
|
{
|
||||||
|
char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
|
||||||
|
htsblk r;
|
||||||
|
|
||||||
#if HTS_FAST_CACHE==0
|
previous_save[0] = '\0';
|
||||||
if (pos < 0) { // pas de mise en cache data, vérifier existence
|
r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
|
||||||
#endif
|
|
||||||
/* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
|
|
||||||
if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) { // fichier final n'existe pas ou est vide!
|
|
||||||
int found = 0;
|
|
||||||
|
|
||||||
/* It is possible that the file has been moved due to changes in build structure */
|
|
||||||
{
|
|
||||||
char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
|
|
||||||
htsblk r;
|
|
||||||
|
|
||||||
previous_save[0] = '\0';
|
|
||||||
r =
|
|
||||||
cache_readex(opt, cache, adr, fil, /*head */ NULL,
|
|
||||||
/*bound to back[p] (temporary) */
|
/*bound to back[p] (temporary) */
|
||||||
back[p].location_buffer, previous_save, /*ro */
|
back[p].location_buffer, previous_save, /*ro */
|
||||||
1);
|
1);
|
||||||
/* Is supposed to be on disk only */
|
/* Is supposed to be on disk only */
|
||||||
if (r.is_write && previous_save[0] != '\0') {
|
if (r.is_write && previous_save[0] != '\0') {
|
||||||
/* Exists, but with another (old) filename: rename (almost) silently */
|
/* Exists, but with another (old) filename: rename (almost)
|
||||||
if (strcmp(previous_save, save) != 0
|
* silently */
|
||||||
&& fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
|
if (strcmp(previous_save, save) != 0 &&
|
||||||
rename(fconv(catbuff, sizeof(catbuff), previous_save),
|
fexist_utf8(
|
||||||
fconv(catbuff2, sizeof(catbuff2), save));
|
fconv(catbuff, sizeof(catbuff), previous_save))) {
|
||||||
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
|
rename(fconv(catbuff, sizeof(catbuff), previous_save),
|
||||||
found = 1;
|
fconv(catbuff2, sizeof(catbuff2), save));
|
||||||
hts_log_print(opt, LOG_DEBUG,
|
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
|
||||||
"File '%s' has been renamed since last mirror to '%s' ; applying changes",
|
found = 1;
|
||||||
previous_save, save);
|
hts_log_print(opt, LOG_DEBUG,
|
||||||
} else {
|
"File '%s' has been renamed since last "
|
||||||
hts_log_print(opt, LOG_ERROR,
|
"mirror to '%s' ; applying changes",
|
||||||
"Could not rename '%s' to '%s' ; will have to retransfer it",
|
previous_save, save);
|
||||||
previous_save, save);
|
} else {
|
||||||
}
|
hts_log_print(opt, LOG_ERROR,
|
||||||
|
"Could not rename '%s' to '%s' ; will have "
|
||||||
|
"to retransfer it",
|
||||||
|
previous_save, save);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
back[p].location_buffer[0] = '\0';
|
|
||||||
}
|
}
|
||||||
|
back[p].location_buffer[0] = '\0';
|
||||||
|
}
|
||||||
|
|
||||||
/* Not found ? */
|
/* Not found ? */
|
||||||
if (!found) {
|
if (!found) {
|
||||||
#if HTS_FAST_CACHE
|
// invalidate: gone from disk, force a refetch
|
||||||
hash_pos_return = 0;
|
hash_pos_return = 0;
|
||||||
#else
|
if (opt->norecatch) {
|
||||||
a = NULL;
|
if (!fexist_utf8(fconv(
|
||||||
#endif
|
catbuff, sizeof(catbuff),
|
||||||
// dévalider car non présent sur disque dans structure originale!!!
|
save))) { // declared but missing: user erased it
|
||||||
// sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
|
FILE *fp =
|
||||||
// en être sûr
|
FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
|
||||||
if (opt->norecatch) { // tester norecatch
|
|
||||||
if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) { // fichier existe pas mais déclaré: on l'a effacé
|
|
||||||
FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
|
|
||||||
|
|
||||||
if (fp)
|
if (fp)
|
||||||
fclose(fp);
|
fclose(fp);
|
||||||
hts_log_print(opt, LOG_WARNING,
|
|
||||||
"Previous file '%s' not found (erased by user ?), ignoring: %s%s",
|
|
||||||
save, back[p].url_adr, back[p].url_fil);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
hts_log_print(opt, LOG_WARNING,
|
hts_log_print(opt, LOG_WARNING,
|
||||||
"Previous file '%s' not found (erased by user ?), recatching: %s%s",
|
"Previous file '%s' not found (erased by "
|
||||||
|
"user ?), ignoring: %s%s",
|
||||||
save, back[p].url_adr, back[p].url_fil);
|
save, back[p].url_adr, back[p].url_fil);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
hts_log_print(opt, LOG_WARNING,
|
||||||
|
"Previous file '%s' not found (erased by user "
|
||||||
|
"?), recatching: %s%s",
|
||||||
|
save, back[p].url_adr, back[p].url_fil);
|
||||||
}
|
}
|
||||||
} // fsize() <= 0
|
}
|
||||||
#if HTS_FAST_CACHE==0
|
} // fsize() <= 0
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
//
|
//
|
||||||
} else {
|
} else {
|
||||||
#if HTS_FAST_CACHE
|
|
||||||
hash_pos_return = 0;
|
hash_pos_return = 0;
|
||||||
#else
|
|
||||||
a = NULL;
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Existe pas en cache, ou bien pas de cache présent
|
if (hash_pos_return) { // in cache, with data
|
||||||
#if HTS_FAST_CACHE
|
|
||||||
if (hash_pos_return) { // OK existe en cache (et données aussi)!
|
|
||||||
#else
|
|
||||||
if (a != NULL) { // OK existe en cache (et données aussi)!
|
|
||||||
#endif
|
|
||||||
const int cache_is_prioritary = cache->type == 1
|
const int cache_is_prioritary = cache->type == 1
|
||||||
|| opt->state.stop != 0;
|
|| opt->state.stop != 0;
|
||||||
if (cache_is_prioritary) { // cache prioritaire (pas de test if-modified..)
|
if (cache_is_prioritary) { // cache prioritaire (pas de test if-modified..)
|
||||||
@@ -1898,7 +1864,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
|
/* Not in cache ; maybe in temporary cache ? Warning: non-movable
|
||||||
|
"url_sav" */
|
||||||
else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
|
else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
|
||||||
const off_t file_size = fsize_utf8(itemback->url_sav);
|
const off_t file_size = fsize_utf8(itemback->url_sav);
|
||||||
|
|
||||||
|
|||||||
@@ -491,11 +491,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
|||||||
hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
|
hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
|
||||||
/* avoid errors on data entries */
|
/* avoid errors on data entries */
|
||||||
if (adr[0] == '/' && adr[1] == '/' && adr[2] == '[') {
|
if (adr[0] == '/' && adr[1] == '/' && adr[2] == '[') {
|
||||||
#if HTS_FAST_CACHE
|
|
||||||
hash_pos_return = 0;
|
hash_pos_return = 0;
|
||||||
#else
|
|
||||||
a = NULL;
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (hash_pos_return != 0) {
|
if (hash_pos_return != 0) {
|
||||||
|
|||||||
@@ -76,9 +76,6 @@ Please visit our Website: http://www.httrack.com
|
|||||||
// always direct-to-disk (0/1)
|
// always direct-to-disk (0/1)
|
||||||
#define HTS_DIRECTDISK_ALWAYS 1
|
#define HTS_DIRECTDISK_ALWAYS 1
|
||||||
|
|
||||||
// fast cache (build hash table)
|
|
||||||
#define HTS_FAST_CACHE 1
|
|
||||||
|
|
||||||
// le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
|
// le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
|
||||||
// valide)
|
// valide)
|
||||||
#define GT_ENDS_COMMENT 1
|
#define GT_ENDS_COMMENT 1
|
||||||
|
|||||||
@@ -134,10 +134,6 @@ RUN_CALLBACK0(opt, end); \
|
|||||||
back_delete_all(opt, &cache, sback); \
|
back_delete_all(opt, &cache, sback); \
|
||||||
back_free(&sback); \
|
back_free(&sback); \
|
||||||
checkrobots_free(&robots); \
|
checkrobots_free(&robots); \
|
||||||
if (cache.use) { \
|
|
||||||
freet(cache.use); \
|
|
||||||
cache.use = NULL; \
|
|
||||||
} \
|
|
||||||
if (cache.zipOutput) { \
|
if (cache.zipOutput) { \
|
||||||
zipClose(cache.zipOutput, \
|
zipClose(cache.zipOutput, \
|
||||||
"Created by HTTrack Website Copier/" HTTRACK_VERSION); \
|
"Created by HTTrack Website Copier/" HTTRACK_VERSION); \
|
||||||
|
|||||||
@@ -194,7 +194,6 @@ struct cache_back {
|
|||||||
int type;
|
int type;
|
||||||
int ro; /**< read-only: no new cache is written */
|
int ro; /**< read-only: no new cache is written */
|
||||||
|
|
||||||
char *use; /**< in-memory list of cached adr+fil keys */
|
|
||||||
FILE *lst; /**< file list, used for purge */
|
FILE *lst; /**< file list, used for purge */
|
||||||
FILE *txt; /**< human-readable file list (info) */
|
FILE *txt; /**< human-readable file list (info) */
|
||||||
char lastmodified[256];
|
char lastmodified[256];
|
||||||
|
|||||||
Reference in New Issue
Block a user