mirror of
https://github.com/xroche/httrack.git
synced 2026-07-26 18:49:39 +03:00
Compare commits
1 Commits
feat/sitem
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
bef7120423 |
@@ -163,20 +163,8 @@ the index" problems disappear.</p>
|
||||
<tr><td><tt>--near (-n)</tt></td><td>Also fetch non-HTML files "near" a followed link, such as an image linked from a page you kept but hosted elsewhere.</td></tr>
|
||||
<tr><td><tt>--ext-depth (-%e)</tt></td><td>How many levels of external links to follow once the crawl leaves your scope (default 0).</td></tr>
|
||||
<tr><td><tt>--test (-t)</tt></td><td>Also HEAD-test links that fall outside the scope, which are normally refused, without downloading them: a way to see what scope is excluding.</td></tr>
|
||||
<tr><td><tt>--sitemap (-%m), --sitemap-url URL (-%mu)</tt></td><td>Also take start URLs from the site's sitemap, for pages nothing links to. Off by default.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>Link-following only finds what something links to. Anything a site publishes
|
||||
solely in its sitemap is invisible to HTTrack unless you ask for it.
|
||||
<tt>--sitemap</tt> reads the start host's <tt>robots.txt</tt> for
|
||||
<tt>Sitemap:</tt> lines and falls back to <tt>/sitemap.xml</tt>;
|
||||
<tt>--sitemap-url</tt> names one directly. Nested <tt>sitemapindex</tt> files
|
||||
and gzipped <tt>.xml.gz</tt> sitemaps are followed. The URLs found become start
|
||||
URLs with the full depth budget, but they still go through your filters and
|
||||
scope rules, so a sitemap cannot widen a crawl you deliberately narrowed. It is
|
||||
off by default because a sitemap can list thousands of pages nothing links
|
||||
to.</p>
|
||||
|
||||
<p>The single most common surprise is "only the home page came down." That is
|
||||
usually not a scope option at all: it is an off-host redirect. A start URL of
|
||||
<tt>http://example.com/</tt> that redirects to <tt>https://www.example.com/</tt>
|
||||
|
||||
@@ -87,8 +87,8 @@ offline browser : copy websites to a local directory</p>
|
||||
--host-control[=N]</b> ] [ <b>-%P,
|
||||
--extended-parsing[=N]</b> ] [ <b>-n, --near</b> ] [ <b>-t,
|
||||
--test</b> ] [ <b>-%L, --list</b> ] [ <b>-%S, --urllist</b>
|
||||
] [ <b>-%m, --sitemap</b> ] [ <b>-NN, --structure[=N]</b> ]
|
||||
[ <b>-%N, --delayed-type-check</b> ] [ <b>-%D,
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
@@ -575,19 +575,6 @@ URL per line) (--list <param>)</p></td></tr>
|
||||
|
||||
<p><file> add all scan rules located in this text
|
||||
file (one scan rule per line) (--urllist <param>)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%m</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>seed the crawl from the site’s sitemap (robots.txt
|
||||
Sitemap:, then /sitemap.xml); --sitemap-url URL names one
|
||||
explicitly (--sitemap)</p></td></tr>
|
||||
</table>
|
||||
|
||||
<h3>Build options:
|
||||
|
||||
@@ -132,17 +132,6 @@ ${listid:robots:LISTDEF_8}
|
||||
</select>
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="sitemap" ${checked:sitemap}
|
||||
title='${html:LANG_SITEMAPTIP}' onMouseOver="info('${html:LANG_SITEMAPTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_SITEMAP}
|
||||
<br><br>
|
||||
|
||||
${LANG_SITEMAPURL}
|
||||
<input name="sitemapurl" value="${sitemapurl}" size="40"
|
||||
title='${html:LANG_SITEMAPURLTIP}' onMouseOver="info('${html:LANG_SITEMAPURLTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
>
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="updhack" ${checked:updhack}
|
||||
title='${html:LANG_I1k}' onMouseOver="info('${html:LANG_I1k}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I62b}
|
||||
|
||||
@@ -141,8 +141,6 @@ ${do:copy:KeepSlashes:keepslashes}
|
||||
${do:copy:KeepQueryOrder:keepqueryorder}
|
||||
${do:copy:StripQuery:stripquery}
|
||||
${do:copy:StoreAllInCache:cache2}
|
||||
${do:copy:Sitemap:sitemap}
|
||||
${do:copy:SitemapUrl:sitemapurl}
|
||||
${do:copy:Warc:warc}
|
||||
${do:copy:WarcFile:warcfile}
|
||||
${do:copy:LogType:logtype}
|
||||
|
||||
@@ -188,8 +188,6 @@ ${/* -m<n> resets the html limit, so the bare form must precede the -m,<n> one *
|
||||
${test:toler:--tolerant}
|
||||
${test:http10:--http-10}
|
||||
${test:cache2:--store-all-in-cache}
|
||||
${test:sitemap:--sitemap}
|
||||
${test:sitemapurl:--sitemap-url "}${html:sitemapurl}${test:sitemapurl:"}
|
||||
${test:warc:--warc}
|
||||
${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
|
||||
${test:norecatch:--do-not-recatch}
|
||||
@@ -242,8 +240,6 @@ KeepSlashes=${ztest:keepslashes:0:1}
|
||||
KeepQueryOrder=${ztest:keepqueryorder:0:1}
|
||||
StripQuery=${stripquery}
|
||||
StoreAllInCache=${ztest:cache2:0:1}
|
||||
Sitemap=${ztest:sitemap:0:1}
|
||||
SitemapUrl=${sitemapurl}
|
||||
Warc=${ztest:warc:0:1}
|
||||
WarcFile=${warcfile}
|
||||
LogType=${logtype}
|
||||
|
||||
8
lang.def
8
lang.def
@@ -1042,11 +1042,3 @@ LANG_WARCFILE
|
||||
WARC archive name:
|
||||
LANG_WARCFILETIP
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
LANG_SITEMAP
|
||||
Seed the crawl from the site's sitemap
|
||||
LANG_SITEMAPTIP
|
||||
Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
|
||||
LANG_SITEMAPURL
|
||||
Sitemap address:
|
||||
LANG_SITEMAPURLTIP
|
||||
Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
|
||||
|
||||
@@ -1012,11 +1012,3 @@ WARC archive name:
|
||||
WARC archive name:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Seed the crawl from the site's sitemap
|
||||
Seed the crawl from the site's sitemap
|
||||
Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
|
||||
Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
|
||||
Sitemap address:
|
||||
Sitemap address:
|
||||
Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
|
||||
Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
|
||||
|
||||
@@ -1012,11 +1012,3 @@ WARC archive name:
|
||||
Nom de l'archive WARC :
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nom de base optionnel pour l'archive WARC ; laissez vide pour le générer automatiquement dans le répertoire de sortie.
|
||||
Seed the crawl from the site's sitemap
|
||||
Partir du plan de site (sitemap)
|
||||
Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
|
||||
Lire le plan de site (lignes Sitemap: de robots.txt, puis /sitemap.xml) et ajouter chaque URL listée comme adresse de départ.
|
||||
Sitemap address:
|
||||
Adresse du plan de site :
|
||||
Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
|
||||
Adresse d'un plan de site à lire au lieu de sonder le site ; laissez vide pour sonder robots.txt puis /sitemap.xml.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "26 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "23 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -36,7 +36,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-t, \-\-test\fR ]
|
||||
[ \fB\-%L, \-\-list\fR ]
|
||||
[ \fB\-%S, \-\-urllist\fR ]
|
||||
[ \fB\-%m, \-\-sitemap\fR ]
|
||||
[ \fB\-NN, \-\-structure[=N]\fR ]
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
@@ -188,8 +187,6 @@ test all URLs (even forbidden ones) (\-\-test)
|
||||
<file> add all URL located in this text file (one URL per line) (\-\-list <param>)
|
||||
.IP \-%S
|
||||
<file> add all scan rules located in this text file (one scan rule per line) (\-\-urllist <param>)
|
||||
.IP \-%m
|
||||
seed the crawl from the site's sitemap (robots.txt Sitemap:, then /sitemap.xml); \-\-sitemap\-url URL names one explicitly (\-\-sitemap)
|
||||
.SS Build options:
|
||||
.IP \-NN
|
||||
structure type (0 *original structure, 1+: see below) (\-\-structure[=N])
|
||||
|
||||
@@ -66,7 +66,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htswarc.c htssitemap.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmd5.c htscodec.c htswarc.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
@@ -77,7 +77,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htssitemap.h htsproxy.h htszlib.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htsproxy.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
htscharset.h punycode.h htsencoding.h \
|
||||
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
|
||||
|
||||
@@ -114,10 +114,6 @@ const char *hts_optalias[][4] = {
|
||||
"strip [host/pattern=]key1,key2,... from URLs"},
|
||||
{"cookies-file", "-%K", "param1",
|
||||
"load extra cookies from a Netscape cookies.txt"},
|
||||
{"sitemap", "-%m", "single",
|
||||
"seed the crawl from the start host's sitemap (robots.txt, then "
|
||||
"/sitemap.xml)"},
|
||||
{"sitemap-url", "-%mu", "param1", "seed the crawl from this sitemap URL"},
|
||||
{"warc", "-%r", "single", "write an ISO-28500 WARC/1.1 archive of the crawl"},
|
||||
{"warc-file", "-%rf", "param1", "write a WARC archive to the given base name"},
|
||||
{"warc-max-size", "-%rs", "param1",
|
||||
|
||||
@@ -39,7 +39,6 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
/* File defs */
|
||||
#include "htscore.h"
|
||||
#include "htssitemap.h"
|
||||
#include "htswarc.h"
|
||||
|
||||
/* specific definitions */
|
||||
@@ -944,21 +943,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
heap_top()->premier = heap_top_index(); // premier lien, objet-père=objet
|
||||
heap_top()->precedent = heap_top_index(); // lien précédent
|
||||
|
||||
/* --sitemap: queue the sitemap probe just after the seeds, so its URLs are
|
||||
injected before the crawl gets far. */
|
||||
if (opt->sitemap || StringNotEmpty(opt->sitemap_url)) {
|
||||
char BIGSTK first[HTS_URLMAXSIZE * 2];
|
||||
const char *const eol = strchr(primary, '\n');
|
||||
const size_t len = eol != NULL ? (size_t) (eol - primary) : 0;
|
||||
|
||||
first[0] = '\0';
|
||||
if (len > 0 && len < sizeof(first)) {
|
||||
memcpy(first, primary, len);
|
||||
first[len] = '\0';
|
||||
}
|
||||
hts_sitemap_seed(opt, first);
|
||||
}
|
||||
|
||||
// Initialiser cache
|
||||
{
|
||||
opt->state._hts_in_html_parsing = 4;
|
||||
@@ -1625,28 +1609,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
/* Load file and decode if necessary, after redirect check. */
|
||||
LOAD_IN_MEMORY_IF_NECESSARY();
|
||||
|
||||
/* Sitemap document: turn its <loc> URLs into top-level seeds. They go
|
||||
through htsAddLink, so the wizard's filters and scope rules decide, and
|
||||
this link's max depth leaves them the full budget. */
|
||||
if (opt->sitemap_state != NULL &&
|
||||
hts_sitemap_pending(opt, urladr(), urlfil())) {
|
||||
htsmoduleStruct BIGSTK smstr;
|
||||
|
||||
memset(&smstr, 0, sizeof(smstr));
|
||||
smstr.opt = opt;
|
||||
smstr.sback = sback;
|
||||
smstr.cache = &cache;
|
||||
smstr.hashptr = hashptr;
|
||||
smstr.numero_passe = numero_passe;
|
||||
smstr.ptr_ = &ptr;
|
||||
smstr.addLink = htsAddLink;
|
||||
smstr.url_host = urladr();
|
||||
smstr.url_file = urlfil();
|
||||
smstr.mime = r.contenttype;
|
||||
hts_sitemap_ingest(opt, &smstr, urladr(), urlfil(), r.adr,
|
||||
r.adr != NULL && r.size > 0 ? (size_t) r.size : 0);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------
|
||||
// ok, fichier chargé localement
|
||||
// ------------------------------------------------------
|
||||
@@ -2294,7 +2256,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// ending
|
||||
usercommand(opt, 0, NULL, NULL, NULL, NULL);
|
||||
warc_close_opt(opt);
|
||||
hts_sitemap_free(opt);
|
||||
|
||||
// désallocation mémoire & buffers
|
||||
XH_uninit;
|
||||
@@ -3682,11 +3643,6 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
to->warc_cdx = from->warc_cdx;
|
||||
to->warc_wacz = from->warc_wacz;
|
||||
|
||||
if (from->sitemap)
|
||||
to->sitemap = from->sitemap;
|
||||
if (StringNotEmpty(from->sitemap_url))
|
||||
StringCopyS(to->sitemap_url, from->sitemap_url);
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
to->pause_max_ms = from->pause_max_ms;
|
||||
|
||||
@@ -1795,26 +1795,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
break;
|
||||
case 'm': // sitemap / sitemap-url: seed the crawl from sitemaps
|
||||
if (*(com + 1) == 'u') { // --sitemap-url URL: explicit sitemap
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option sitemap-url needs a blank space and a URL");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
|
||||
HTS_PANIC_PRINTF("Sitemap URL too long");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
StringCopy(opt->sitemap_url, argv[na]);
|
||||
} else { // --sitemap: robots.txt probe, then /sitemap.xml
|
||||
opt->sitemap = HTS_TRUE;
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
|
||||
@@ -525,8 +525,6 @@ void help(const char *app, int more) {
|
||||
(" %L <file> add all URL located in this text file (one URL per line)");
|
||||
infomsg
|
||||
(" %S <file> add all scan rules located in this text file (one scan rule per line)");
|
||||
infomsg(" %m seed the crawl from the site's sitemap (robots.txt Sitemap:, "
|
||||
"then /sitemap.xml); --sitemap-url URL names one explicitly");
|
||||
infomsg("");
|
||||
infomsg("Build options:");
|
||||
infomsg(" NN structure type (0 *original structure, 1+: see below)");
|
||||
|
||||
@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com
|
||||
// Fichier librairie .c
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htssitemap.h"
|
||||
#include "htswarc.h"
|
||||
|
||||
/* specific definitions */
|
||||
@@ -6011,7 +6010,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->strip_query, "");
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->warc_file, "");
|
||||
StringCopy(opt->sitemap_url, "");
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
@@ -6165,8 +6163,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
|
||||
StringFree(opt->cookies_file);
|
||||
StringFree(opt->why_url);
|
||||
StringFree(opt->warc_file);
|
||||
StringFree(opt->sitemap_url);
|
||||
hts_sitemap_free(opt); /* backstop: httpmirror's early-return paths */
|
||||
|
||||
StringFree(opt->path_html);
|
||||
StringFree(opt->path_html_utf8);
|
||||
|
||||
@@ -547,10 +547,6 @@ struct httrackp {
|
||||
archive. Tail: ABI */
|
||||
hts_boolean warc_wacz; /**< --wacz: package archive+index+pages as a WACZ zip
|
||||
(implies --warc + --warc-cdx). Tail: ABI */
|
||||
hts_boolean sitemap; /**< --sitemap: probe the start host's robots.txt for
|
||||
Sitemap: lines, else /sitemap.xml. Tail: ABI */
|
||||
String sitemap_url; /**< --sitemap-url: sitemap to ingest. Tail: ABI */
|
||||
void *sitemap_state; /**< live sitemap ingestion state, or NULL. Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
|
||||
@@ -58,7 +58,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htssniff.h"
|
||||
#include "htscodec.h"
|
||||
#include "htsproxy.h"
|
||||
#include "htssitemap.h"
|
||||
#include "htswarc.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
@@ -3575,166 +3574,6 @@ static int st_robots(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Collect the URLs a sitemap scan hands out. */
|
||||
typedef struct sm_collect {
|
||||
int n;
|
||||
char url[8][HTS_URLMAXSIZE];
|
||||
} sm_collect;
|
||||
|
||||
static hts_boolean sm_take(void *arg, const char *url) {
|
||||
sm_collect *const c = (sm_collect *) arg;
|
||||
|
||||
if (c->n < (int) (sizeof(c->url) / sizeof(c->url[0])))
|
||||
strcpybuff(c->url[c->n], url);
|
||||
c->n++;
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Scan `doc` off a heap buffer with no NUL terminator, so a read past the
|
||||
declared size is an ASan error rather than a silent pass. */
|
||||
static int sm_scan(const char *doc, int maxurls, hts_boolean *is_index,
|
||||
sm_collect *out) {
|
||||
const size_t len = strlen(doc);
|
||||
char *raw = malloct(len);
|
||||
int n;
|
||||
|
||||
memset(out, 0, sizeof(*out));
|
||||
assertf(raw != NULL);
|
||||
memcpy(raw, doc, len);
|
||||
n = hts_sitemap_scan(raw, len, maxurls, is_index, sm_take, out);
|
||||
freet(raw);
|
||||
return n;
|
||||
}
|
||||
|
||||
static int st_sitemap(httrackp *opt, int argc, char **argv) {
|
||||
sm_collect c;
|
||||
hts_boolean idx;
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
/* A urlset yields its <loc> URLs, in order, unescaped. */
|
||||
assertf(sm_scan("<?xml version=\"1.0\"?><urlset>"
|
||||
"<url><loc>http://h.test/a.html</loc></url>"
|
||||
"<url><loc> https://h.test/b?x=1&y=2\n </loc></url>"
|
||||
"</urlset>",
|
||||
100, &idx, &c) == 2);
|
||||
assertf(!idx);
|
||||
assertf(strcmp(c.url[0], "http://h.test/a.html") == 0);
|
||||
assertf(strcmp(c.url[1], "https://h.test/b?x=1&y=2") == 0);
|
||||
|
||||
/* A sitemapindex is flagged: its URLs are child sitemaps, not pages. */
|
||||
assertf(sm_scan("<sitemapindex><sitemap><loc>http://h.test/s2.xml.gz</loc>"
|
||||
"</sitemap></sitemapindex>",
|
||||
100, &idx, &c) == 1);
|
||||
assertf(idx);
|
||||
|
||||
/* Root element decides even when the other name appears later as text. */
|
||||
assertf(sm_scan("<urlset><url><loc>http://h.test/a</loc></url>"
|
||||
"<!-- sitemapindex --></urlset>",
|
||||
100, &idx, &c) == 1);
|
||||
assertf(!idx);
|
||||
|
||||
/* Numeric character references, decimal and hex, decode to ASCII. */
|
||||
assertf(sm_scan("<urlset><loc>http://h.test/a?b=c</loc></urlset>",
|
||||
100, &idx, &c) == 1);
|
||||
assertf(strcmp(c.url[0], "http://h.test/a?b=c") == 0);
|
||||
|
||||
/* A reference outside printable ASCII stays verbatim, not a control byte. */
|
||||
assertf(sm_scan("<urlset><loc>http://h.test/a b</loc></urlset>", 100,
|
||||
&idx, &c) == 1);
|
||||
assertf(strcmp(c.url[0], "http://h.test/a b") == 0);
|
||||
|
||||
/* <location> is not <loc>. */
|
||||
assertf(sm_scan("<urlset><location>http://h.test/a</location></urlset>", 100,
|
||||
&idx, &c) == 0);
|
||||
|
||||
/* Rejected: relative, non-http scheme, embedded space, empty. */
|
||||
assertf(sm_scan("<urlset><loc>/a.html</loc><loc>ftp://h.test/a</loc>"
|
||||
"<loc>javascript:alert(1)</loc>"
|
||||
"<loc>http://h.test/a b</loc><loc></loc></urlset>",
|
||||
100, &idx, &c) == 0);
|
||||
|
||||
/* The URL length bound: one under fits, exactly at it is dropped rather than
|
||||
truncated into a different URL. */
|
||||
{
|
||||
char BIGSTK doc[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK url[HTS_URLMAXSIZE + 1];
|
||||
size_t i;
|
||||
|
||||
strcpybuff(url, "http://h.test/");
|
||||
for (i = strlen(url); i < HTS_URLMAXSIZE - 1; i++)
|
||||
url[i] = 'a';
|
||||
url[i] = '\0';
|
||||
snprintf(doc, sizeof(doc), "<urlset><loc>%s</loc></urlset>", url);
|
||||
assertf(sm_scan(doc, 100, &idx, &c) == 1);
|
||||
|
||||
url[i] = 'a';
|
||||
url[i + 1] = '\0';
|
||||
snprintf(doc, sizeof(doc), "<urlset><loc>%s</loc></urlset>", url);
|
||||
assertf(sm_scan(doc, 100, &idx, &c) == 0);
|
||||
}
|
||||
|
||||
/* The URL cap stops the scan. */
|
||||
assertf(sm_scan("<urlset><loc>http://h.test/1</loc><loc>http://h.test/2</loc>"
|
||||
"<loc>http://h.test/3</loc></urlset>",
|
||||
2, &idx, &c) == 2);
|
||||
|
||||
/* An unterminated <loc> at end of buffer must not read past it. */
|
||||
assertf(sm_scan("<urlset><loc>http://h.test/a", 100, &idx, &c) == 0);
|
||||
assertf(sm_scan("<urlset><lo", 100, &idx, &c) == 0);
|
||||
|
||||
#if HTS_USEZLIB
|
||||
/* A gzip-framed document is decompressed before scanning. */
|
||||
{
|
||||
const char *const xml =
|
||||
"<urlset><url><loc>http://h.test/gz.html</loc></url></urlset>";
|
||||
uLongf zlen = compressBound((uLong) strlen(xml)) + 32;
|
||||
char *z = malloct((size_t) zlen);
|
||||
z_stream zs;
|
||||
|
||||
assertf(z != NULL);
|
||||
memset(&zs, 0, sizeof(zs));
|
||||
assertf(deflateInit2(&zs, 9, Z_DEFLATED, 16 + MAX_WBITS, 8,
|
||||
Z_DEFAULT_STRATEGY) == Z_OK);
|
||||
zs.next_in = (const Bytef *) xml;
|
||||
zs.avail_in = (uInt) strlen(xml);
|
||||
zs.next_out = (Bytef *) z;
|
||||
zs.avail_out = (uInt) zlen;
|
||||
assertf(deflate(&zs, Z_FINISH) == Z_STREAM_END);
|
||||
zlen = (uLongf) zs.total_out;
|
||||
deflateEnd(&zs);
|
||||
|
||||
memset(&c, 0, sizeof(c));
|
||||
assertf(hts_sitemap_scan(z, (size_t) zlen, 100, &idx, sm_take, &c) == 1);
|
||||
assertf(strcmp(c.url[0], "http://h.test/gz.html") == 0);
|
||||
|
||||
/* Truncated gzip: refused, not scanned as plain text. */
|
||||
memset(&c, 0, sizeof(c));
|
||||
assertf(hts_sitemap_scan(z, 4, 100, &idx, sm_take, &c) == -1);
|
||||
freet(z);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* robots.txt: only Sitemap: records, comments stripped, case-insensitive,
|
||||
and group-independent (no User-agent line needed). */
|
||||
memset(&c, 0, sizeof(c));
|
||||
{
|
||||
const char *const txt = "User-agent: *\nDisallow: /x\n"
|
||||
"SITEMAP: http://h.test/s1.xml # first\n"
|
||||
"Sitemap: /relative.xml\n"
|
||||
"Sitemapper: http://h.test/no.xml\n"
|
||||
"Sitemap:\thttps://h.test/s2.xml\n";
|
||||
|
||||
assertf(hts_sitemap_scan_robots(txt, strlen(txt), 100, sm_take, &c) == 2);
|
||||
assertf(strcmp(c.url[0], "http://h.test/s1.xml") == 0);
|
||||
assertf(strcmp(c.url[1], "https://h.test/s2.xml") == 0);
|
||||
}
|
||||
|
||||
printf("sitemap self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Connected stream pair over loopback; Windows has no socketpair(). */
|
||||
static int st_socketpair(T_SOC sv[2]) {
|
||||
struct sockaddr_in sa;
|
||||
@@ -5343,8 +5182,6 @@ static const struct selftest_entry {
|
||||
st_contentcodings},
|
||||
{"robots", "", "robots.txt RFC 9309 Allow/Disallow precedence self-test",
|
||||
st_robots},
|
||||
{"sitemap", "",
|
||||
"sitemap <loc> extraction, caps and robots.txt Sitemap:", st_sitemap},
|
||||
{"ftp-line", "", "get_ftp_line bounds a hostile FTP reply line",
|
||||
st_ftpline},
|
||||
{"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
|
||||
|
||||
544
src/htssitemap.c
544
src/htssitemap.c
@@ -1,544 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: sitemap ingestion (sitemaps.org 0.9) */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#define HTS_INTERNAL_BYTECODE
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htssitemap.h"
|
||||
|
||||
#include "htsbase.h"
|
||||
#include "htscodec.h"
|
||||
#include "htslib.h"
|
||||
#include "htsrobots.h"
|
||||
#include "htssafe.h"
|
||||
#include "htstools.h"
|
||||
#include "htszlib.h"
|
||||
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
|
||||
/* One queued sitemap document. `is_robots` marks the robots.txt probe, whose
|
||||
Sitemap: lines feed this same list. */
|
||||
typedef struct sitemap_doc {
|
||||
char adr[HTS_URLMAXSIZE];
|
||||
char fil[HTS_URLMAXSIZE];
|
||||
int level;
|
||||
hts_boolean is_robots;
|
||||
hts_boolean done;
|
||||
struct sitemap_doc *next;
|
||||
} sitemap_doc;
|
||||
|
||||
struct hts_sitemap_state {
|
||||
sitemap_doc *docs;
|
||||
int ndocs; /* documents queued, capped by HTS_SITEMAP_MAX_DOCS */
|
||||
int nurls; /* URLs seeded, capped by HTS_SITEMAP_MAX_URLS_TOTAL */
|
||||
hts_boolean fallback_done; /* the /sitemap.xml fallback was already queued */
|
||||
};
|
||||
typedef struct hts_sitemap_state hts_sitemap_state;
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
/* Document parsing (no engine state: fuzzable and self-testable) */
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Decode the five XML predefined entities plus ASCII-printable numeric refs
|
||||
in-place; anything else is left verbatim. Never grows the string. */
|
||||
static void sitemap_unescape(char *s) {
|
||||
char *r = s, *w = s;
|
||||
|
||||
while (*r != '\0') {
|
||||
if (*r != '&') {
|
||||
*w++ = *r++;
|
||||
continue;
|
||||
}
|
||||
{
|
||||
char *const semi = strchr(r + 1, ';');
|
||||
const size_t len = semi != NULL ? (size_t) (semi - (r + 1)) : 0;
|
||||
int c = -1;
|
||||
|
||||
if (len == 0 || len > 8) {
|
||||
*w++ = *r++;
|
||||
continue;
|
||||
}
|
||||
if (len == 3 && strncmp(r + 1, "amp", 3) == 0)
|
||||
c = '&';
|
||||
else if (len == 2 && strncmp(r + 1, "lt", 2) == 0)
|
||||
c = '<';
|
||||
else if (len == 2 && strncmp(r + 1, "gt", 2) == 0)
|
||||
c = '>';
|
||||
else if (len == 4 && strncmp(r + 1, "quot", 4) == 0)
|
||||
c = '"';
|
||||
else if (len == 4 && strncmp(r + 1, "apos", 4) == 0)
|
||||
c = '\'';
|
||||
else if (r[1] == '#') {
|
||||
const int hex = (r[2] == 'x' || r[2] == 'X');
|
||||
const char *p = r + (hex ? 3 : 2);
|
||||
long v = 0;
|
||||
|
||||
if (p < semi) {
|
||||
for (; p < semi; p++) {
|
||||
const int d =
|
||||
hex ? (isxdigit((unsigned char) *p)
|
||||
? (isdigit((unsigned char) *p)
|
||||
? *p - '0'
|
||||
: (tolower((unsigned char) *p) - 'a' + 10))
|
||||
: -1)
|
||||
: (isdigit((unsigned char) *p) ? *p - '0' : -1);
|
||||
|
||||
if (d < 0) {
|
||||
v = -1;
|
||||
break;
|
||||
}
|
||||
v = v * (hex ? 16 : 10) + d;
|
||||
if (v > 0x7e)
|
||||
break;
|
||||
}
|
||||
/* Only ASCII printables: a URL has no business carrying anything
|
||||
else, and a wider decode would let a reference smuggle in a
|
||||
control character. */
|
||||
if (v >= 0x20 && v <= 0x7e)
|
||||
c = (int) v;
|
||||
}
|
||||
}
|
||||
if (c < 0) {
|
||||
*w++ = *r++;
|
||||
} else {
|
||||
*w++ = (char) c;
|
||||
r = semi + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
*w = '\0';
|
||||
}
|
||||
|
||||
/* Accept only an absolute http(s) URL with no space or control byte. */
|
||||
static hts_boolean sitemap_url_ok(const char *url) {
|
||||
const char *p;
|
||||
|
||||
if (!strfield(url, "http://") && !strfield(url, "https://"))
|
||||
return HTS_FALSE;
|
||||
for (p = url; *p != '\0'; p++) {
|
||||
if ((unsigned char) *p <= ' ' || (unsigned char) *p == 0x7f)
|
||||
return HTS_FALSE;
|
||||
}
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Skip to the character after the next '>' at or after p, or NULL. */
|
||||
static const char *sitemap_tag_end(const char *p, const char *end) {
|
||||
while (p < end && *p != '>')
|
||||
p++;
|
||||
return p < end ? p + 1 : NULL;
|
||||
}
|
||||
|
||||
/* Bounded substring search: the document may hold NUL bytes. */
|
||||
static const char *sitemap_memstr(const char *p, size_t len,
|
||||
const char *needle) {
|
||||
const size_t nlen = strlen(needle);
|
||||
|
||||
if (nlen == 0 || len < nlen)
|
||||
return NULL;
|
||||
for (; len >= nlen; p++, len--) {
|
||||
if (*p == *needle && memcmp(p, needle, nlen) == 0)
|
||||
return p;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Decompress a gzip-framed body into a fresh buffer bounded by both the
|
||||
absolute cap and the codec ratio budget. Returns NULL on failure. */
|
||||
static char *sitemap_gunzip(const char *body, size_t size, size_t *outsize) {
|
||||
const LLint budget = hts_codec_maxout((LLint) size);
|
||||
size_t cap = budget < (LLint) HTS_SITEMAP_MAX_BYTES
|
||||
? (size_t) budget
|
||||
: (size_t) HTS_SITEMAP_MAX_BYTES;
|
||||
char *out;
|
||||
size_t n;
|
||||
|
||||
if (cap == 0)
|
||||
return NULL;
|
||||
out = malloct(cap + 1);
|
||||
if (out == NULL)
|
||||
return NULL;
|
||||
n = hts_zhead(body, size, out, cap);
|
||||
if (n == 0) {
|
||||
freet(out);
|
||||
return NULL;
|
||||
}
|
||||
out[n] = '\0';
|
||||
*outsize = n;
|
||||
return out;
|
||||
}
|
||||
|
||||
int hts_sitemap_scan(const char *body, size_t size, int maxurls,
|
||||
hts_boolean *is_index, hts_sitemap_handler handler,
|
||||
void *arg) {
|
||||
char *unpacked = NULL;
|
||||
const char *doc;
|
||||
const char *end;
|
||||
const char *p;
|
||||
int n = 0;
|
||||
|
||||
if (is_index != NULL)
|
||||
*is_index = HTS_FALSE;
|
||||
if (body == NULL || size < 2 || handler == NULL)
|
||||
return 0;
|
||||
|
||||
/* A .xml.gz body arrives raw here: Content-Encoding gzip was already undone
|
||||
upstream, so only the gzip container is left to peel. */
|
||||
if ((unsigned char) body[0] == 0x1f && (unsigned char) body[1] == 0x8b) {
|
||||
unpacked = sitemap_gunzip(body, size, &size);
|
||||
if (unpacked == NULL)
|
||||
return -1;
|
||||
doc = unpacked;
|
||||
} else {
|
||||
if (size > (size_t) HTS_SITEMAP_MAX_BYTES)
|
||||
size = (size_t) HTS_SITEMAP_MAX_BYTES;
|
||||
doc = body;
|
||||
}
|
||||
end = doc + size;
|
||||
|
||||
/* Whichever root element comes first classifies the document; the handler
|
||||
reads the verdict before the first URL, so it must be set up front. */
|
||||
if (is_index != NULL) {
|
||||
const char *const idx = sitemap_memstr(doc, size, "<sitemapindex");
|
||||
const char *const set = sitemap_memstr(doc, size, "<urlset");
|
||||
|
||||
if (idx != NULL && (set == NULL || idx < set))
|
||||
*is_index = HTS_TRUE;
|
||||
}
|
||||
|
||||
for (p = doc; n < maxurls;) {
|
||||
const char *loc = sitemap_memstr(p, (size_t) (end - p), "<loc");
|
||||
const char *val;
|
||||
const char *stop;
|
||||
size_t len;
|
||||
char BIGSTK url[HTS_URLMAXSIZE];
|
||||
|
||||
if (loc == NULL)
|
||||
break;
|
||||
/* "<loc>" or "<loc xmlns:..>", never "<location>" */
|
||||
if (loc + 4 >= end || (loc[4] != '>' && !isspace((unsigned char) loc[4]))) {
|
||||
p = loc + 4;
|
||||
continue;
|
||||
}
|
||||
val = sitemap_tag_end(loc + 4, end);
|
||||
if (val == NULL)
|
||||
break;
|
||||
for (stop = val; stop < end && *stop != '<'; stop++)
|
||||
;
|
||||
/* No closing tag: the document is truncated (cut short, or clipped by the
|
||||
decompression cap), so the last value may be a partial URL. Drop it. */
|
||||
if (stop == end)
|
||||
break;
|
||||
p = stop;
|
||||
while (val < stop && isspace((unsigned char) *val))
|
||||
val++;
|
||||
while (stop > val && isspace((unsigned char) *(stop - 1)))
|
||||
stop--;
|
||||
len = (size_t) (stop - val);
|
||||
/* Overflow-safe: the untrusted length alone against the room left. */
|
||||
if (len == 0 || len >= sizeof(url))
|
||||
continue;
|
||||
memcpy(url, val, len);
|
||||
url[len] = '\0';
|
||||
sitemap_unescape(url);
|
||||
if (!sitemap_url_ok(url))
|
||||
continue;
|
||||
n++;
|
||||
if (!handler(arg, url))
|
||||
break;
|
||||
}
|
||||
|
||||
if (unpacked != NULL)
|
||||
freet(unpacked);
|
||||
return n;
|
||||
}
|
||||
|
||||
int hts_sitemap_scan_robots(const char *body, size_t size, int maxurls,
|
||||
hts_sitemap_handler handler, void *arg) {
|
||||
size_t bptr = 0;
|
||||
int n = 0;
|
||||
|
||||
if (body == NULL || handler == NULL)
|
||||
return 0;
|
||||
while (bptr < size && n < maxurls) {
|
||||
char BIGSTK line[HTS_URLMAXSIZE];
|
||||
char *comm;
|
||||
char *a;
|
||||
|
||||
bptr += binput(body + bptr, line, sizeof(line) - 2);
|
||||
comm = strchr(line, '#');
|
||||
if (comm != NULL)
|
||||
*comm = '\0';
|
||||
if (!strfield(line, "sitemap:"))
|
||||
continue;
|
||||
a = line + 8;
|
||||
while (is_realspace(*a))
|
||||
a++;
|
||||
{
|
||||
size_t l = strlen(a);
|
||||
|
||||
while (l > 0 && is_realspace(a[l - 1]))
|
||||
a[--l] = '\0';
|
||||
}
|
||||
if (!sitemap_url_ok(a))
|
||||
continue;
|
||||
n++;
|
||||
if (!handler(arg, a))
|
||||
break;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
/* Engine glue */
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
static hts_sitemap_state *sitemap_state(httrackp *opt) {
|
||||
if (opt->sitemap_state == NULL)
|
||||
opt->sitemap_state = calloct(1, sizeof(hts_sitemap_state));
|
||||
return (hts_sitemap_state *) opt->sitemap_state;
|
||||
}
|
||||
|
||||
static sitemap_doc *sitemap_find(httrackp *opt, const char *adr,
|
||||
const char *fil) {
|
||||
hts_sitemap_state *const st = (hts_sitemap_state *) opt->sitemap_state;
|
||||
sitemap_doc *d;
|
||||
|
||||
if (st == NULL)
|
||||
return NULL;
|
||||
for (d = st->docs; d != NULL; d = d->next) {
|
||||
if (strfield2(d->adr, adr) && strcmp(d->fil, fil) == 0)
|
||||
return d;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Queue a document and record its link with save="" so the body stays in
|
||||
memory: a sitemap is ingested, never mirrored. Top priority so its URLs get
|
||||
the full depth budget through htsAddLink. */
|
||||
static hts_boolean sitemap_queue(httrackp *opt, const char *adr,
|
||||
const char *fil, int level,
|
||||
hts_boolean is_robots) {
|
||||
hts_sitemap_state *const st = sitemap_state(opt);
|
||||
sitemap_doc *d;
|
||||
|
||||
if (st == NULL)
|
||||
return HTS_FALSE;
|
||||
if (st->ndocs >= HTS_SITEMAP_MAX_DOCS || level > HTS_SITEMAP_MAX_LEVEL) {
|
||||
hts_log_print(opt, LOG_WARNING, "Sitemap: cap reached, skipping %s%s", adr,
|
||||
fil);
|
||||
return HTS_FALSE;
|
||||
}
|
||||
if (strlen(adr) >= sizeof(d->adr) || strlen(fil) >= sizeof(d->fil))
|
||||
return HTS_FALSE;
|
||||
if (sitemap_find(opt, adr, fil) != NULL)
|
||||
return HTS_FALSE;
|
||||
d = calloct(1, sizeof(sitemap_doc));
|
||||
if (d == NULL)
|
||||
return HTS_FALSE;
|
||||
strcpybuff(d->adr, adr);
|
||||
strcpybuff(d->fil, fil);
|
||||
d->level = level;
|
||||
d->is_robots = is_robots;
|
||||
d->next = st->docs;
|
||||
st->docs = d;
|
||||
st->ndocs++;
|
||||
|
||||
if (!hts_record_link(opt, adr, fil, "", "", "", NULL))
|
||||
return HTS_FALSE;
|
||||
heap_top()->testmode = 0;
|
||||
heap_top()->link_import = 0;
|
||||
heap_top()->depth = opt->depth + 1;
|
||||
heap_top()->pass2 = 0;
|
||||
heap_top()->retry = opt->retry;
|
||||
heap_top()->premier = heap_top_index();
|
||||
heap_top()->precedent = heap_top_index();
|
||||
hts_log_print(opt, LOG_INFO, "Sitemap: queued %s%s", adr, fil);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
void hts_sitemap_seed(httrackp *opt, const char *starturl) {
|
||||
char BIGSTK url[HTS_URLMAXSIZE * 2];
|
||||
lien_adrfil af;
|
||||
|
||||
if (StringNotEmpty(opt->sitemap_url)) {
|
||||
if (strlen(StringBuff(opt->sitemap_url)) >= sizeof(url)) {
|
||||
hts_log_print(opt, LOG_ERROR, "Sitemap URL too long");
|
||||
} else {
|
||||
strcpybuff(url, StringBuff(opt->sitemap_url));
|
||||
if (strstr(url, ":/") == NULL)
|
||||
hts_log_print(opt, LOG_ERROR, "Sitemap URL must be absolute: %s", url);
|
||||
else if (ident_url_absolute(url, &af) >= 0)
|
||||
(void) sitemap_queue(opt, af.adr, af.fil, 0, HTS_FALSE);
|
||||
}
|
||||
}
|
||||
/* --sitemap: probe the start host's robots.txt, whose Sitemap: lines decide
|
||||
whether the /sitemap.xml fallback is needed. */
|
||||
if (!opt->sitemap || starturl == NULL || starturl[0] == '\0' ||
|
||||
strlen(starturl) >= sizeof(url))
|
||||
return;
|
||||
strcpybuff(url, starturl);
|
||||
if (ident_url_absolute(url, &af) < 0)
|
||||
return;
|
||||
if (sitemap_queue(opt, af.adr, "/robots.txt", 0, HTS_TRUE)) {
|
||||
/* Claim the host so the parser does not queue robots.txt a second time. */
|
||||
if (opt->robotsptr != NULL)
|
||||
(void) checkrobots_set((robots_wizard *) opt->robotsptr, af.adr, "");
|
||||
}
|
||||
}
|
||||
|
||||
hts_boolean hts_sitemap_pending(httrackp *opt, const char *adr,
|
||||
const char *fil) {
|
||||
const sitemap_doc *const d = sitemap_find(opt, adr, fil);
|
||||
|
||||
return d != NULL && !d->done ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Handler context: seeding URLs from one document. */
|
||||
typedef struct sitemap_ingest_ctx {
|
||||
httrackp *opt;
|
||||
htsmoduleStruct *str;
|
||||
const char *adr; /* host of the document being ingested */
|
||||
int level;
|
||||
hts_boolean is_index;
|
||||
} sitemap_ingest_ctx;
|
||||
|
||||
/* A <loc> of a <urlset>: hand it to the wizard as a top-level seed. */
|
||||
static hts_boolean sitemap_seed_url(void *arg, const char *url) {
|
||||
sitemap_ingest_ctx *const c = (sitemap_ingest_ctx *) arg;
|
||||
hts_sitemap_state *const st = sitemap_state(c->opt);
|
||||
char BIGSTK buff[HTS_URLMAXSIZE];
|
||||
|
||||
if (st == NULL || st->nurls >= HTS_SITEMAP_MAX_URLS_TOTAL) {
|
||||
hts_log_print(c->opt, LOG_WARNING,
|
||||
"Sitemap: URL cap reached, ignoring the rest");
|
||||
return HTS_FALSE;
|
||||
}
|
||||
/* Both scanners bound the URL below this, but strcpybuff aborts rather than
|
||||
truncating, so never let hostile input reach it unchecked. */
|
||||
if (strlen(url) >= sizeof(buff))
|
||||
return HTS_TRUE;
|
||||
st->nurls++;
|
||||
strcpybuff(buff, url);
|
||||
(void) htsAddLink(c->str, buff);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* A <loc> of a <sitemapindex>, or a robots.txt Sitemap: line. Cross-host
|
||||
children are dropped: a hostile sitemap must not aim the fetcher elsewhere.
|
||||
*/
|
||||
static hts_boolean sitemap_seed_child(void *arg, const char *url) {
|
||||
sitemap_ingest_ctx *const c = (sitemap_ingest_ctx *) arg;
|
||||
char BIGSTK buff[HTS_URLMAXSIZE];
|
||||
lien_adrfil af;
|
||||
|
||||
if (strlen(url) >= sizeof(buff))
|
||||
return HTS_TRUE;
|
||||
strcpybuff(buff, url);
|
||||
if (ident_url_absolute(buff, &af) < 0)
|
||||
return HTS_TRUE;
|
||||
if (!strfield2(af.adr, c->adr)) {
|
||||
hts_log_print(c->opt, LOG_WARNING,
|
||||
"Sitemap: ignoring off-host child sitemap %s%s", af.adr,
|
||||
af.fil);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
(void) sitemap_queue(c->opt, af.adr, af.fil, c->level + 1, HTS_FALSE);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* hts_sitemap_scan classifies the document before the first callback, so the
|
||||
urlset/sitemapindex split can be decided here. */
|
||||
static hts_boolean sitemap_seed_any(void *arg, const char *url) {
|
||||
sitemap_ingest_ctx *const c = (sitemap_ingest_ctx *) arg;
|
||||
|
||||
return c->is_index ? sitemap_seed_child(arg, url)
|
||||
: sitemap_seed_url(arg, url);
|
||||
}
|
||||
|
||||
void hts_sitemap_ingest(httrackp *opt, htsmoduleStruct *str, const char *adr,
|
||||
const char *fil, const char *body, size_t size) {
|
||||
sitemap_doc *const d = sitemap_find(opt, adr, fil);
|
||||
sitemap_ingest_ctx ctx;
|
||||
int n;
|
||||
|
||||
if (d == NULL || d->done)
|
||||
return;
|
||||
d->done = HTS_TRUE;
|
||||
ctx.opt = opt;
|
||||
ctx.str = str;
|
||||
ctx.adr = adr;
|
||||
ctx.level = d->level;
|
||||
ctx.is_index = HTS_FALSE;
|
||||
|
||||
if (d->is_robots) {
|
||||
hts_sitemap_state *const st = sitemap_state(opt);
|
||||
|
||||
n = body != NULL ? hts_sitemap_scan_robots(body, size, HTS_SITEMAP_MAX_DOCS,
|
||||
sitemap_seed_child, &ctx)
|
||||
: 0;
|
||||
/* Fall back to the well-known location only when robots.txt named none. */
|
||||
if (n == 0 && st != NULL && !st->fallback_done) {
|
||||
st->fallback_done = HTS_TRUE;
|
||||
(void) sitemap_queue(opt, adr, "/sitemap.xml", 0, HTS_FALSE);
|
||||
}
|
||||
hts_log_print(opt, LOG_INFO, "Sitemap: %d sitemap(s) declared in %s%s", n,
|
||||
adr, fil);
|
||||
return;
|
||||
}
|
||||
|
||||
n = hts_sitemap_scan(body, size, HTS_SITEMAP_MAX_URLS_DOC, &ctx.is_index,
|
||||
sitemap_seed_any, &ctx);
|
||||
if (n < 0) {
|
||||
hts_log_print(opt, LOG_ERROR, "Sitemap: could not decompress %s%s", adr,
|
||||
fil);
|
||||
return;
|
||||
}
|
||||
hts_log_print(opt, LOG_NOTICE, "Sitemap: %d URL(s) added from %s%s", n, adr,
|
||||
fil);
|
||||
}
|
||||
|
||||
void hts_sitemap_free(httrackp *opt) {
|
||||
hts_sitemap_state *const st = (hts_sitemap_state *) opt->sitemap_state;
|
||||
|
||||
if (st == NULL)
|
||||
return;
|
||||
while (st->docs != NULL) {
|
||||
sitemap_doc *const next = st->docs->next;
|
||||
|
||||
freet(st->docs);
|
||||
st->docs = next;
|
||||
}
|
||||
freet(opt->sitemap_state);
|
||||
opt->sitemap_state = NULL;
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* HTTrack sitemap ingestion (sitemaps.org 0.9). Internal, not installed.
|
||||
Reads <urlset>/<sitemapindex> documents, plain or gzip-framed, and feeds
|
||||
their <loc> URLs to the crawl as top-level seeds. The whole input is
|
||||
attacker-controlled, so every entry point below is capped. */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_SITEMAP_DEFH
|
||||
#define HTS_SITEMAP_DEFH
|
||||
|
||||
#include "htsdefines.h"
|
||||
#include "htsopt.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Caps. sitemaps.org allows 50000 URLs and 50 MB uncompressed per document;
|
||||
the byte cap sits above that so a conformant sitemap always fits. */
|
||||
#define HTS_SITEMAP_MAX_URLS_DOC 50000 /* <loc> per document */
|
||||
#define HTS_SITEMAP_MAX_URLS_TOTAL 200000 /* <loc> per mirror */
|
||||
#define HTS_SITEMAP_MAX_DOCS 256 /* documents per mirror */
|
||||
#define HTS_SITEMAP_MAX_LEVEL 4 /* sitemapindex nesting */
|
||||
#define HTS_SITEMAP_MAX_BYTES (64 * 1024 * 1024) /* decompressed document */
|
||||
|
||||
/* Per-URL handler; returning HTS_FALSE stops the scan. */
|
||||
typedef hts_boolean (*hts_sitemap_handler)(void *arg, const char *url);
|
||||
|
||||
/* Scan one sitemap document, plain or gzip-framed, handing every acceptable
|
||||
absolute http(s) <loc> URL to `handler`. Stops after `maxurls` URLs, or when
|
||||
the handler refuses. `is_index` (optional) reports a <sitemapindex>, whose
|
||||
URLs are child sitemaps rather than pages. Returns the number of URLs handed
|
||||
out, or -1 when the document could not be decompressed within the caps. */
|
||||
int hts_sitemap_scan(const char *body, size_t size, int maxurls,
|
||||
hts_boolean *is_index, hts_sitemap_handler handler,
|
||||
void *arg);
|
||||
|
||||
/* Same, over a NUL-terminated robots.txt body: hands out the "Sitemap:" URLs
|
||||
(a group-independent record in RFC 9309, so no user-agent grouping). */
|
||||
int hts_sitemap_scan_robots(const char *body, size_t size, int maxurls,
|
||||
hts_sitemap_handler handler, void *arg);
|
||||
|
||||
/* --- Engine glue (needs a live httrackp). --- */
|
||||
|
||||
/* Queue the first sitemap document of the mirror: the explicit --sitemap-url,
|
||||
or the start host's /robots.txt probe for --sitemap. `starturl` is the first
|
||||
command-line seed. No-op when neither option is set. */
|
||||
void hts_sitemap_seed(httrackp *opt, const char *starturl);
|
||||
|
||||
/* HTS_TRUE when (adr,fil) is a queued sitemap document awaiting ingestion. */
|
||||
hts_boolean hts_sitemap_pending(httrackp *opt, const char *adr,
|
||||
const char *fil);
|
||||
|
||||
/* Ingest a fetched sitemap document (or the robots.txt probe): seed its URLs
|
||||
through the wizard via htsAddLink, and queue nested sitemaps. `str` supplies
|
||||
the parser context of the document being processed. */
|
||||
void hts_sitemap_ingest(httrackp *opt, htsmoduleStruct *str, const char *adr,
|
||||
const char *fil, const char *body, size_t size);
|
||||
|
||||
/* Release the ingestion state held in opt (NULL-safe, idempotent). */
|
||||
void hts_sitemap_free(httrackp *opt);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -138,7 +138,6 @@
|
||||
<ClCompile Include="htswrap.c" />
|
||||
<ClCompile Include="htszlib.c" />
|
||||
<ClCompile Include="htswarc.c" />
|
||||
<ClCompile Include="htssitemap.c" />
|
||||
<ClCompile Include="md5.c" />
|
||||
<ClCompile Include="minizip\ioapi.c" />
|
||||
<ClCompile Include="minizip\iowin32.c" />
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Sitemap parser self-test: <loc> extraction, entity decoding, URL and length
|
||||
# rejections, the URL cap, gzip framing and robots.txt Sitemap: records.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
out=$(httrack -O /dev/null '-#test=sitemap')
|
||||
echo "$out"
|
||||
test "$out" = "sitemap self-test OK"
|
||||
@@ -67,10 +67,10 @@ test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
|
||||
srvpid=$(sed -n 's/^PID=//p' "${srvlog}") # absent on Windows
|
||||
|
||||
# htsserver resolves the posted paths itself, so hand it native ones.
|
||||
"${python}" - "${url}" "$(nativepath "${work}/websites")" "$(nativepath "${proj}")" <<'PY' || fail "browse-link checks failed"
|
||||
"${python}" - "${url}" "$(nativepath "${work}/websites")" <<'PY' || fail "browse-link checks failed"
|
||||
import re, sys, urllib.error, urllib.parse, urllib.request
|
||||
|
||||
url, base, proj = sys.argv[1].rstrip("/"), sys.argv[2], sys.argv[3]
|
||||
url, base = sys.argv[1].rstrip("/"), sys.argv[2]
|
||||
rc = 0
|
||||
|
||||
|
||||
@@ -100,13 +100,14 @@ except urllib.error.HTTPError as e:
|
||||
check(e.code == 404, "/website/ is bound to a project (got %d)" % e.code)
|
||||
|
||||
# Point the server at the mirror the way step4.html does; a body without the
|
||||
# session id is refused outright.
|
||||
# session id is refused outright. Only a profile save records the served root,
|
||||
# so post one; command_do=save runs no crawl.
|
||||
sid = re.search(r'name="sid" value="([0-9a-f]+)"', finished)
|
||||
if sid is None:
|
||||
print("FAIL: no sid in the rendered form")
|
||||
sys.exit(1)
|
||||
fields = [("sid", sid.group(1)), ("path", base), ("projname", "proj"),
|
||||
("projpath", proj + "/")]
|
||||
("command", "httrack"), ("command_do", "save"), ("winprofile", "x")]
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="")) for k, v in fields)
|
||||
urllib.request.urlopen(urllib.request.Request(
|
||||
url + "/server/step4.html", data=body.encode("latin-1"), method="POST"),
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# --sitemap seeds the crawl from robots.txt -> sitemapindex -> gzipped urlset.
|
||||
# start.html links to nothing, so orphan*.html can only arrive through the
|
||||
# sitemap; deep1.html proves the seeds keep a full depth budget under -r2, and
|
||||
# the off-host page and child sitemap must both be refused (--errors 0).
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
crawl() { bash "$top_srcdir/tests/local-crawl.sh" "$@"; }
|
||||
|
||||
# robots.txt Sitemap: -> index -> .xml.gz, and the seeds behave like -r2 seeds.
|
||||
# --rerun also walks the update path over a mirror holding sitemap seeds.
|
||||
crawl --errors 0 --rerun \
|
||||
--found 'sitemapdir/orphan1.html' \
|
||||
--found 'sitemapdir/orphan2.html' \
|
||||
--found 'sitemapdir/deep1.html' \
|
||||
httrack 'BASEURL/sitemapdir/start.html' --sitemap -r2
|
||||
|
||||
# Negative control: without the option nothing but the start page is reached.
|
||||
crawl --errors 0 \
|
||||
--found 'sitemapdir/start.html' \
|
||||
--not-found 'sitemapdir/orphan1.html' \
|
||||
--not-found 'sitemapdir/orphan2.html' \
|
||||
httrack 'BASEURL/sitemapdir/start.html' -r2
|
||||
|
||||
# Sitemap URLs are not a filter bypass: a -*orphan2* rule still rejects one.
|
||||
crawl --errors 0 \
|
||||
--found 'sitemapdir/orphan1.html' \
|
||||
--not-found 'sitemapdir/orphan2.html' \
|
||||
httrack 'BASEURL/sitemapdir/start.html' --sitemap -r2 '-*orphan2*'
|
||||
|
||||
# A robots.txt naming no sitemap falls back to the well-known /sitemap.xml.
|
||||
# The test server drops its Sitemap: record for this User-Agent.
|
||||
crawl --errors 0 \
|
||||
--found 'sitemapdir/orphan1.html' \
|
||||
--log-not-found 'sitemapdir/index.xml' \
|
||||
httrack 'BASEURL/sitemapdir/start.html' --sitemap -r2 -F 'nositemap-agent'
|
||||
|
||||
# --sitemap-url names the document directly, skipping the robots.txt probe.
|
||||
crawl --errors 0 \
|
||||
--found 'sitemapdir/orphan1.html' \
|
||||
--found 'sitemapdir/orphan2.html' \
|
||||
--log-not-found 'sitemapdir/index.xml' \
|
||||
httrack 'BASEURL/sitemapdir/start.html' -r2 \
|
||||
--sitemap-url 'BASEURL/sitemapdir/pages.xml.gz'
|
||||
|
||||
# An unfetchable sitemap is not fatal: the crawl still completes.
|
||||
crawl --found 'sitemapdir/start.html' \
|
||||
httrack 'BASEURL/sitemapdir/start.html' -r2 \
|
||||
--sitemap-url 'BASEURL/sitemapdir/missing.xml'
|
||||
@@ -89,7 +89,6 @@ TESTS = \
|
||||
01_engine-warc-surt.test \
|
||||
01_engine-xfread.test \
|
||||
01_zlib-acceptencoding.test \
|
||||
01_engine-sitemap.test \
|
||||
01_zlib-warc.test \
|
||||
01_zlib-warc-cdx.test \
|
||||
01_zlib-warc-wacz.test \
|
||||
@@ -183,7 +182,6 @@ TESTS = \
|
||||
82_webhttrack-browse-links.test \
|
||||
83_webhttrack-argescape.test \
|
||||
84_webhttrack-mirror-verbatim.test \
|
||||
85_webhttrack-projpath.test \
|
||||
86_local-sitemap.test
|
||||
85_webhttrack-projpath.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -542,16 +542,8 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
return self.fail_cookie(name)
|
||||
self.send_html("\tThis is the secret.")
|
||||
|
||||
# A User-Agent carrying NO_SITEMAP_UA gets a robots.txt with no Sitemap:
|
||||
# record, so a test can drive the /sitemap.xml fallback instead.
|
||||
NO_SITEMAP_UA = "nositemap"
|
||||
|
||||
def route_robots(self):
|
||||
# The Sitemap: record is group-independent; only --sitemap acts on it.
|
||||
body = "User-agent: *\nDisallow:\n"
|
||||
if self.NO_SITEMAP_UA not in (self.headers.get("User-Agent") or ""):
|
||||
body += f"Sitemap: http://{self.headers.get('Host')}/sitemapdir/index.xml\n"
|
||||
body = body.encode()
|
||||
body = b"User-agent: *\nDisallow:\n"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
@@ -559,49 +551,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
# --- sitemap ingestion (issue #712) ------------------------------------
|
||||
# start.html links to nothing, so orphan*.html are reachable only through
|
||||
# the sitemap. deep1.html proves the seeds keep a full depth budget; the
|
||||
# off-host page <loc> must be dropped by the travel scope, and the off-host
|
||||
# child sitemap by the ingester's same-host rule. The index is served both
|
||||
# from /sitemapdir/ (named by robots.txt) and from the well-known
|
||||
# /sitemap.xml (the fallback).
|
||||
|
||||
def route_sitemap_index(self):
|
||||
host = self.headers.get("Host")
|
||||
self.send_raw(
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
|
||||
f"<sitemap><loc>http://{host}/sitemapdir/pages.xml.gz</loc></sitemap>"
|
||||
"<sitemap><loc>http://sitemap-offhost.invalid/s.xml</loc></sitemap>"
|
||||
"</sitemapindex>\n".encode(),
|
||||
"application/xml",
|
||||
)
|
||||
|
||||
def route_sitemap_pages(self):
|
||||
host = self.headers.get("Host")
|
||||
xml = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
|
||||
f"<url><loc>http://{host}/sitemapdir/orphan1.html</loc></url>"
|
||||
f"<url><loc>http://{host}/sitemapdir/orphan2.html</loc></url>"
|
||||
"<url><loc>http://sitemap-offhost.invalid/x.html</loc></url>"
|
||||
"</urlset>\n"
|
||||
).encode()
|
||||
self.send_raw(gzip.compress(xml), "application/x-gzip")
|
||||
|
||||
def route_sitemap_start(self):
|
||||
self.send_html("\tNothing links to the sitemap pages.")
|
||||
|
||||
def route_sitemap_orphan1(self):
|
||||
self.send_html('\t<a href="deep1.html">deeper</a>')
|
||||
|
||||
def route_sitemap_orphan2(self):
|
||||
self.send_html("\tSecond orphan.")
|
||||
|
||||
def route_sitemap_deep1(self):
|
||||
self.send_html("\tOne level below an orphan.")
|
||||
|
||||
# --- type/extension matrix (issue #267 family) -------------------------
|
||||
|
||||
def send_raw(self, body, content_type, extra_headers=()):
|
||||
@@ -1629,13 +1578,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/gated/index.php": route_gated_index,
|
||||
"/gated/secret.php": route_gated_secret,
|
||||
"/robots.txt": route_robots,
|
||||
"/sitemapdir/index.xml": route_sitemap_index,
|
||||
"/sitemap.xml": route_sitemap_index,
|
||||
"/sitemapdir/pages.xml.gz": route_sitemap_pages,
|
||||
"/sitemapdir/start.html": route_sitemap_start,
|
||||
"/sitemapdir/orphan1.html": route_sitemap_orphan1,
|
||||
"/sitemapdir/orphan2.html": route_sitemap_orphan2,
|
||||
"/sitemapdir/deep1.html": route_sitemap_deep1,
|
||||
"/warcgz/index.html": route_warcgz_index,
|
||||
"/warcgz/page.html": route_warcgz_page,
|
||||
"/warcgz/data.bin": route_warcgz_data,
|
||||
|
||||
@@ -46,14 +46,12 @@ cat >"$stubdir/x-www-browser" <<EOF
|
||||
echo "stub browser invoked with: \$1" >&2
|
||||
# Also fetch an option page and require a rendered title='' tooltip: proves the
|
||||
# option template expands and the \${html:} filter escapes into the attribute.
|
||||
# option9/option8 additionally prove the WARC and sitemap controls render.
|
||||
# option9 additionally proves the WARC control renders with its expanded label.
|
||||
opturl="\${1%/}/server/option2.html"
|
||||
warcurl="\${1%/}/server/option9.html"
|
||||
smurl="\${1%/}/server/option8.html"
|
||||
if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html &&
|
||||
opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='" &&
|
||||
warc="\$(curl -fsSL --max-time 20 "\$warcurl")" && printf '%s' "\$warc" | grep -qaF 'name="warcfile"' && printf '%s' "\$warc" | grep -qaF WARC &&
|
||||
sm="\$(curl -fsSL --max-time 20 "\$smurl")" && printf '%s' "\$sm" | grep -qaF 'name="sitemapurl"' && printf '%s' "\$sm" | grep -qaF 'name="sitemap"'; then
|
||||
warc="\$(curl -fsSL --max-time 20 "\$warcurl")" && printf '%s' "\$warc" | grep -qaF 'name="warcfile"' && printf '%s' "\$warc" | grep -qaF WARC; then
|
||||
echo PASS >"$marker"
|
||||
else
|
||||
echo "FAIL: unexpected response from \$1" >"$marker"
|
||||
|
||||
Reference in New Issue
Block a user