mirror of
https://github.com/xroche/httrack.git
synced 2026-07-17 22:29:55 +03:00
Compare commits
7 Commits
master
...
fix-expand
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a22b7a0e54 | ||
|
|
5944feed9d | ||
|
|
223a8fc27a | ||
|
|
de09705732 | ||
|
|
96b0555522 | ||
|
|
9bbfa39d2a | ||
|
|
b10c6eaf5c |
12
.github/workflows/windows-build.yml
vendored
12
.github/workflows/windows-build.yml
vendored
@@ -158,21 +158,13 @@ jobs:
|
||||
|
||||
pass=0 fail=0 skip=0 failed="" skipped=""
|
||||
for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
|
||||
*_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
|
||||
60_crawl-log-salvage.test; do
|
||||
*_local-*.test 13_crawl_proxy_https.test 58_watchdog.test; do
|
||||
rc=0
|
||||
run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
case "$rc" in
|
||||
0) pass=$((pass + 1)); echo "PASS $t" ;;
|
||||
77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
|
||||
124)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (timed out, tree killed)"
|
||||
# Re-running a wedge traced would just hang again: salvage the
|
||||
# killed crawl's own logs into the artifact instead.
|
||||
dump_crawl_logs >>"$t.log"
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
124) fail=$((fail + 1)) failed="$failed $t"; echo "FAIL $t (timed out, tree killed)" ;;
|
||||
*)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (exit $rc)"
|
||||
|
||||
@@ -23,7 +23,7 @@ pt:7
|
||||
ro:25
|
||||
ru:8
|
||||
sk:20
|
||||
sl:24
|
||||
si:24
|
||||
sv:17
|
||||
tr:10
|
||||
uk:22
|
||||
|
||||
@@ -930,6 +930,7 @@ Server terminated
|
||||
Palvelin lopetettu
|
||||
A fatal error has occurred during this mirror
|
||||
Tällä peilillä tapahtui vakava virhe
|
||||
|
||||
Proxy type:
|
||||
Välityspalvelimen tyyppi:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
|
||||
@@ -374,7 +374,7 @@ Create error logging and report files
|
||||
Crea file di log per segnalare errori e informazioni
|
||||
Generate DOS 8-3 filenames ONLY
|
||||
Genera solo nomi di file in formato 8.3
|
||||
Generate ISO9660 filenames ONLY for CDROM medias
|
||||
Generate ISO 9660 filenames ONLY for CDROM medias
|
||||
Genera nomi di file in formato ISO9660 per i CDROM
|
||||
Do not create HTML error pages
|
||||
Non generare pagine d'errore HTML
|
||||
|
||||
@@ -918,7 +918,7 @@ normal\nextended\ndebug
|
||||
normal\nextendido\ncorrigir
|
||||
Download web site(s)\nDownload web site(s) + questions\nGet individual files\nDownload all sites in pages (multiple mirror)\nTest links in pages (bookmark test)\n* Continue interrupted download\n* Update existing download
|
||||
Copiar site(s) da Web\nCopiar site(s) interativos da Web (perguntas)\nReceber arquivos específicos\nCopiar todas as páginas do site (alternação múltipla)\nTestar links nas páginas (testar indicador)\n* Retomar download interrompido\n* Atualizar download existente
|
||||
Relative URI / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
|
||||
Relative URL / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
|
||||
URI Relativa / URL Absoluta (padrão)\nURL Absoluta / URL Absoluta\nURI Absoluta / URL Absoluta\nURL Original / URL Original
|
||||
Open Source offline browser
|
||||
Abrir origem offline no navegador
|
||||
|
||||
@@ -3,7 +3,7 @@ Slovenian
|
||||
LANGUAGE_FILE
|
||||
Slovenian
|
||||
LANGUAGE_ISO
|
||||
sl
|
||||
si
|
||||
LANGUAGE_AUTHOR
|
||||
Jadran Rudec,iur.\r\njrudec@email.si \r\n
|
||||
LANGUAGE_CHARSET
|
||||
|
||||
@@ -113,7 +113,6 @@ HTSEXT_API int hts_main(int argc, char **argv) {
|
||||
}
|
||||
|
||||
static int hts_main_internal(int argc, char **argv, httrackp * opt);
|
||||
static hts_boolean cmdl_shortopt_has(const char *s, char c);
|
||||
|
||||
// Main, récupère les paramètres et appelle le robot
|
||||
HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt) {
|
||||
@@ -305,12 +304,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
hts_get_version_info(opt));
|
||||
return 0;
|
||||
} else {
|
||||
if (strncmp(tmp_argv[0], "--", 2)) { /* not a long option */
|
||||
if (cmdl_shortopt_has(tmp_argv[0], 'q'))
|
||||
opt->quiet = HTS_TRUE; // never ask questions (nohup)
|
||||
if (cmdl_shortopt_has(tmp_argv[0], 'i')) { // doit.log!
|
||||
argv_url = -1;
|
||||
opt->quiet = HTS_TRUE;
|
||||
if (strncmp(tmp_argv[0], "--", 2)) { /* pas */
|
||||
if ((strchr(tmp_argv[0], 'q') != NULL))
|
||||
opt->quiet = 1; // ne pas poser de questions! (nohup par exemple)
|
||||
if ((strchr(tmp_argv[0], 'i') != NULL)) { // doit.log!
|
||||
argv_url = -1; /* forcer */
|
||||
opt->quiet = 1;
|
||||
}
|
||||
} else if (strcmp(tmp_argv[0] + 2, "quiet") == 0) {
|
||||
opt->quiet = 1; // ne pas poser de questions! (nohup par exemple)
|
||||
@@ -425,10 +424,25 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|
||||
} // for
|
||||
|
||||
// path_html is already UTF-8 (argv is UTF-8 on Windows via
|
||||
// hts_argv_utf8), so no re-encoding.
|
||||
// Convert path to UTF-8
|
||||
#ifdef _WIN32
|
||||
{
|
||||
char *const path =
|
||||
hts_convertStringSystemToUTF8(StringBuff(opt->path_html),
|
||||
(int) StringLength(opt->path_html));
|
||||
if (path != NULL) {
|
||||
StringCopy(opt->path_html_utf8, path);
|
||||
free(path);
|
||||
} else {
|
||||
StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
|
||||
StringLength(opt->path_html));
|
||||
}
|
||||
}
|
||||
#else
|
||||
// Assume UTF-8 filesystem.
|
||||
StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
|
||||
StringLength(opt->path_html));
|
||||
#endif
|
||||
|
||||
/* if doit.log exists, or if new URL(s) defined,
|
||||
then DO NOT load standard config files */
|
||||
@@ -2471,12 +2485,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
} */
|
||||
|
||||
// vérifier existence de la structure (path_html/path_log are UTF-8, use
|
||||
// the UTF-8 mkdir path)
|
||||
structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "/"));
|
||||
structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "/"));
|
||||
// vérifier existence de la structure
|
||||
structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "/"));
|
||||
structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "/"));
|
||||
|
||||
// reprise/update
|
||||
if (opt->cache) {
|
||||
@@ -2796,32 +2807,6 @@ int check_path(String * s, char *defaultname) {
|
||||
return return_value;
|
||||
}
|
||||
|
||||
/* Does the short-option cluster s carry c from the main option set (-i, -iC2,
|
||||
-%Mi)? Walked as the parser does below: %, &, @ and # each take the letter
|
||||
after them into another set, so the i of -%i is not the main-set -i. */
|
||||
static hts_boolean cmdl_shortopt_has(const char *s, char c) {
|
||||
const char *com;
|
||||
|
||||
if (s[0] != '-' || s[1] == '-')
|
||||
return HTS_FALSE;
|
||||
for (com = s + 1; *com != '\0'; com++) {
|
||||
switch (*com) {
|
||||
case '%':
|
||||
case '&':
|
||||
case '@':
|
||||
case '#':
|
||||
if (*(com + 1) != '\0')
|
||||
com++; /* skip the other set's letter */
|
||||
break;
|
||||
default:
|
||||
if (*com == c)
|
||||
return HTS_TRUE;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
// détermine si l'argument est une option
|
||||
int cmdl_opt(char *s) {
|
||||
if (s[0] == '-') { // c'est peut être une option
|
||||
|
||||
@@ -402,50 +402,6 @@ int dns_selftests(httrackp *opt) {
|
||||
deletesoc(s);
|
||||
}
|
||||
|
||||
/* A URL port outside 1..65535 must refuse the link, not fold into range and
|
||||
connect elsewhere (#614). *addr_count discriminates: 0 only if refused
|
||||
before the resolve, still 2 for one merely truncated or defaulted. */
|
||||
{
|
||||
/* an empty "dual.test:" means the default port (WHATWG, curl): keep it */
|
||||
static const char *const good[] = {"dual.test:1", "dual.test:80",
|
||||
"dual.test:8080", "dual.test:65535",
|
||||
"dual.test:080", "dual.test:"};
|
||||
/* 65616 and 4294967376 are load-bearing: both wrap to a plausible 80 */
|
||||
static const char *const bad[] = {
|
||||
"dual.test:0", "dual.test:65536", "dual.test:65616",
|
||||
"dual.test:99999", "dual.test:2147483648", "dual.test:4294967296",
|
||||
"dual.test:4294967376", "dual.test:-1", "dual.test:-23437",
|
||||
"dual.test:80x", "dual.test:+80", "dual.test: 80",
|
||||
"dual.test:0x50"};
|
||||
size_t k;
|
||||
|
||||
for (k = 0; k < sizeof(good) / sizeof(good[0]); k++) {
|
||||
htsblk r;
|
||||
int count = -1;
|
||||
T_SOC s;
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
s = newhttp_addr(opt, good[k], &r, -1, 0, 0, &count);
|
||||
CHECK(count == 2); /* accepted: reached the resolve */
|
||||
if (s != INVALID_SOCKET)
|
||||
deletesoc(s);
|
||||
}
|
||||
|
||||
for (k = 0; k < sizeof(bad) / sizeof(bad[0]); k++) {
|
||||
htsblk r;
|
||||
int count = -1;
|
||||
T_SOC s;
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
s = newhttp_addr(opt, bad[k], &r, -1, 0, 0, &count);
|
||||
CHECK(s == INVALID_SOCKET);
|
||||
CHECK(count == 0); /* refused before resolving, not a failed connect */
|
||||
CHECK(strstr(r.msg, "Invalid port") != NULL);
|
||||
if (s != INVALID_SOCKET)
|
||||
deletesoc(s);
|
||||
}
|
||||
}
|
||||
|
||||
/* Connect-fallback decision (consumer of the multi-address list): when a
|
||||
stuck connect should abandon the current address for the next one. */
|
||||
{
|
||||
|
||||
36
src/htslib.c
36
src/htslib.c
@@ -2167,18 +2167,15 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
#endif
|
||||
|
||||
if (a != NULL) {
|
||||
// folding a nonsense port into 0..65535 crawls one neither the link nor
|
||||
// a port filter named; an empty "host:" just means the default (#614)
|
||||
if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
|
||||
if (retour != NULL) {
|
||||
snprintf(retour->msg, sizeof(retour->msg), "Invalid port: %s",
|
||||
a + 1);
|
||||
}
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
int i = -1;
|
||||
|
||||
iadr2[0] = '\0';
|
||||
// the address itself, without the ":port"
|
||||
sscanf(a + 1, "%d", &i);
|
||||
if (i != -1) {
|
||||
port = (unsigned short int) i;
|
||||
}
|
||||
|
||||
// adresse véritable (sans :xx)
|
||||
strncatbuff(iadr2, iadr, (int) (a - iadr));
|
||||
resolve_host = iadr2;
|
||||
}
|
||||
@@ -3751,27 +3748,18 @@ static int proxy_default_port(const char *arg) {
|
||||
return hts_proxy_is_socks(arg) ? 1080 : 8080;
|
||||
}
|
||||
|
||||
hts_boolean hts_parse_url_port(const char *a, int *port) {
|
||||
// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
|
||||
// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
|
||||
static int parse_proxy_port(const char *a, const char *arg) {
|
||||
char *end;
|
||||
long p;
|
||||
|
||||
if (!isdigit((unsigned char) *a)) // strtol would eat a sign or leading space
|
||||
return HTS_FALSE;
|
||||
return proxy_default_port(arg);
|
||||
p = strtol(a, &end, 10);
|
||||
if (*end != '\0' || p < 1 || p > 65535) // ERANGE lands out of range too
|
||||
return HTS_FALSE;
|
||||
*port = (int) p;
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
|
||||
// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
|
||||
static int parse_proxy_port(const char *a, const char *arg) {
|
||||
int port;
|
||||
|
||||
if (!hts_parse_url_port(a, &port))
|
||||
return proxy_default_port(arg);
|
||||
return port;
|
||||
return (int) p;
|
||||
}
|
||||
|
||||
void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
|
||||
@@ -288,12 +288,6 @@ const char *strrchr_limit(const char *s, char c, const char *limit);
|
||||
char *jump_protocol(char *source);
|
||||
const char *jump_protocol_const(const char *source);
|
||||
|
||||
/* Parse a URL's port text "a" (after the ':', up to the end of the string):
|
||||
TRUE and *port set for a bare decimal in 1..65535, else FALSE and *port left
|
||||
alone. Not sscanf("%d"), which range-checks nothing and wraps past INT_MAX.
|
||||
*/
|
||||
hts_boolean hts_parse_url_port(const char *a, int *port);
|
||||
|
||||
/* Split a -P proxy argument "[scheme://][user:pass@]host[:port]" into the proxy
|
||||
host string (scheme and any user:pass kept, for later stripping and auth),
|
||||
written NUL-terminated into name[name_size] (truncated to fit), and the port
|
||||
|
||||
@@ -491,11 +491,14 @@ static int socks5_handshake_stream(httrackp *opt, socks5_stream *st,
|
||||
if ((unsigned char) host[i] < ' ')
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
|
||||
}
|
||||
// the old range check ran after sscanf("%d") had wrapped a huge value into a
|
||||
// plausible port (#614). An empty "host:" stays refused here, unlike the
|
||||
// direct path, as it was before #614.
|
||||
if (portsep != NULL && !hts_parse_url_port(portsep + 1, &port))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
|
||||
if (portsep != NULL) {
|
||||
int p = -1;
|
||||
|
||||
sscanf(portsep + 1, "%d", &p);
|
||||
if (p <= 0 || p > 65535)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
|
||||
port = p;
|
||||
}
|
||||
if (link_has_authorization(proxy_name)) {
|
||||
if (!socks5_credentials(opt, proxy_name, user, sizeof(user), &userlen, pass,
|
||||
sizeof(pass), &passlen, msg, msgsize))
|
||||
|
||||
@@ -1469,25 +1469,6 @@ static int st_socks5(httrackp *opt, int argc, char **argv) {
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test:8443", proxy, &io) == 1);
|
||||
assertf(memcmp(io.sent + io.sent_len - 2, "\x20\xfb", 2) == 0);
|
||||
|
||||
/* a bad origin port is refused before any byte goes out (#614). 4294967376 is
|
||||
the case the old range check could not see: it overflowed the sscanf("%d")
|
||||
into a plausible 80 and passed. 65616 would not prove anything here, since
|
||||
it fits an int and the old check already caught it. */
|
||||
{
|
||||
static const char *const bad[] = {"origin.test:4294967376",
|
||||
"origin.test:80x", "origin.test:+80",
|
||||
"origin.test: 80", "origin.test:8.0"};
|
||||
size_t k;
|
||||
|
||||
for (k = 0; k < sizeof(bad) / sizeof(bad[0]); k++) {
|
||||
len = socks5_reply(script, 0x01, v4, sizeof(v4));
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, bad[k], proxy, &io) == 0);
|
||||
assertf(io.sent_len == 0);
|
||||
}
|
||||
}
|
||||
|
||||
/* credentials: split on the first colon of the escaped userinfo, so %3a stays
|
||||
inside the username and a colon in the password is not a delimiter */
|
||||
{
|
||||
@@ -2477,45 +2458,6 @@ static int st_makeindex(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// hts_buildtopindex takes a system-charset path but verif_backblue() below it
|
||||
// expects utf-8, so on Windows a non-ASCII project dir gets the gifs written to
|
||||
// a mangled twin (issues #216/#217). argv[0] is a writable dir.
|
||||
static int st_topindex(httrackp *opt, int argc, char **argv) {
|
||||
char projdir[HTS_URLMAXSIZE];
|
||||
char path[HTS_URLMAXSIZE + 16]; /* projdir plus a basename */
|
||||
#ifdef _WIN32
|
||||
/* the GUI hands hts_buildtopindex an ANSI path; mimic it. CP1252 'cafe' */
|
||||
static const char *const projName = "caf\xE9";
|
||||
#else
|
||||
/* POSIX system charset is UTF-8 */
|
||||
static const char *const projName = "caf\xC3\xA9";
|
||||
#endif
|
||||
|
||||
assertf(argc >= 1);
|
||||
snprintf(projdir, sizeof(projdir), "%s/%s", argv[0], projName);
|
||||
|
||||
/* structcheck(), not the utf-8 MKDIR family: same charset as buildtopindex */
|
||||
snprintf(path, sizeof(path), "%s/", projdir);
|
||||
assertf(structcheck(path) == 0);
|
||||
|
||||
/* returns 0 here: the dir holds no sub-project, only the gifs matter */
|
||||
(void) hts_buildtopindex(opt, projdir, "");
|
||||
|
||||
/* the gifs must land in the project dir itself, not in a mangled sibling */
|
||||
snprintf(path, sizeof(path), "%s/backblue.gif", projdir);
|
||||
assertf(fexist(path));
|
||||
|
||||
/* raw unlink/rmdir: UNLINK is utf-8 on Windows, these paths aren't */
|
||||
unlink(path);
|
||||
snprintf(path, sizeof(path), "%s/fade.gif", projdir);
|
||||
unlink(path);
|
||||
snprintf(path, sizeof(path), "%s/index.html", projdir);
|
||||
unlink(path);
|
||||
rmdir(projdir);
|
||||
printf("topindex self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Each inplace_escape_*() must equal escape_*() on a copy. */
|
||||
static int st_inplace_escape(httrackp *opt, int argc, char **argv) {
|
||||
/* >255 bytes forces the helper's malloct path, not the stack buffer */
|
||||
@@ -3223,9 +3165,6 @@ static const struct selftest_entry {
|
||||
{"useragent", "", "default User-Agent self-test", st_useragent},
|
||||
{"makeindex", "[dir]", "hts_finish_makeindex footer/refresh self-test",
|
||||
st_makeindex},
|
||||
{"topindex", "[dir]",
|
||||
"hts_buildtopindex charset handling of a non-ASCII project dir",
|
||||
st_topindex},
|
||||
{"inplace-escape", "", "inplace_escape_* vs escape_* equivalence self-test",
|
||||
st_inplace_escape},
|
||||
{"escape-room", "", "HT_ADD_HTMLESCAPED* reservation-factor self-test",
|
||||
|
||||
@@ -164,12 +164,13 @@ HTS_UNUSED static int linputsoc_t(T_SOC soc, char *s, int max, int timeout) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Same contract as hts_gethome(), which is hidden and out of reach from here */
|
||||
static const char *gethomedir(void) {
|
||||
const char *home = getenv("HOME");
|
||||
|
||||
/* An empty $HOME would put the base path and httrack.ini at the root */
|
||||
return strnotempty(home) ? home : ".";
|
||||
if (home)
|
||||
return home;
|
||||
else
|
||||
return ".";
|
||||
}
|
||||
static int linput_cpp(FILE * fp, char *s, int max) {
|
||||
int rlen = 0;
|
||||
|
||||
@@ -896,20 +896,7 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
|
||||
if (fpo) {
|
||||
find_handle h;
|
||||
|
||||
// générer gif. verif_backblue() is utf-8, but our path is the system
|
||||
// charset, so on Windows convert it or the gifs land in a mangled twin
|
||||
// dir (#217). Elsewhere the system charset is already utf-8.
|
||||
#ifdef _WIN32
|
||||
{
|
||||
const char *const base = concat(catbuff, sizeof(catbuff), rpath, "/");
|
||||
char *const base_utf8 =
|
||||
hts_convertStringSystemToUTF8(base, strlen(base));
|
||||
verif_backblue(opt, base_utf8 != NULL ? base_utf8 : base);
|
||||
free(base_utf8);
|
||||
}
|
||||
#else
|
||||
verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/"));
|
||||
#endif
|
||||
verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/")); // générer gif
|
||||
// Header
|
||||
hts_template_format(fpo, toptemplate_header,
|
||||
"<!-- Mirror and index made by HTTrack Website Copier/"
|
||||
|
||||
@@ -107,12 +107,13 @@ HTS_UNUSED static void proxytrack_print_log(const char *severity, const char *fo
|
||||
"<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \
|
||||
"<!-- End Disable IE Friendly HTTP Error Messages -->\r\n"
|
||||
|
||||
/* Same contract as hts_gethome(): proxytrack does not link libhttrack */
|
||||
HTS_UNUSED static const char *gethomedir(void) {
|
||||
const char *home = getenv("HOME");
|
||||
|
||||
/* An empty $HOME would resolve a relative path against the root */
|
||||
return strnotempty(home) ? home : ".";
|
||||
if (home)
|
||||
return home;
|
||||
else
|
||||
return ".";
|
||||
}
|
||||
|
||||
HTS_UNUSED static int linput(FILE * fp, char *s, int max) {
|
||||
|
||||
@@ -33,19 +33,6 @@ function log {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Map a POSIX locale ("zh_TW.UTF-8@euro") to its lang.indexes number, English (1) if unknown
|
||||
function lang_index {
|
||||
local locale=$1 indexes=$2 tag n t
|
||||
tag=$(echo "${locale}" | cut -f1 -d'.' | cut -f1 -d'@' | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z_')
|
||||
# a few languages carry a region-specific entry (zh_tw, pt_br); else use the bare language
|
||||
for t in "${tag}" "${tag%%_*}"; do
|
||||
n=$(grep -E "^${t}:" "${indexes}" | cut -f2 -d':')
|
||||
test -n "${n}" && break
|
||||
done
|
||||
test -n "${n}" || n=1
|
||||
echo "${n}"
|
||||
}
|
||||
|
||||
function launch_browser {
|
||||
log "Launching $1"
|
||||
browser=$1
|
||||
@@ -72,11 +59,13 @@ test -f "${DISTPATH}/lang.indexes" || ! log "Could not find ${DISTPATH}/lang.ind
|
||||
test -d "${DISTPATH}/lang" || ! log "Could not find ${DISTPATH}/lang" || exit 1
|
||||
test -d "${DISTPATH}/html" || ! log "Could not find ${DISTPATH}/html" || exit 1
|
||||
|
||||
# Locale: POSIX precedence, LC_ALL overrides LC_MESSAGES overrides LANG
|
||||
HTSLANG="${LC_ALL}"
|
||||
! test -n "${HTSLANG}" && HTSLANG="${LC_MESSAGES}"
|
||||
# Locale
|
||||
HTSLANG="${LC_MESSAGES}"
|
||||
! test -n "${HTSLANG}" && HTSLANG="${LC_ALL}"
|
||||
! test -n "${HTSLANG}" && HTSLANG="${LANG}"
|
||||
LANGN=$(lang_index "${HTSLANG}" "${DISTPATH}/lang.indexes")
|
||||
HTSLANG="$(echo "$LANG" | cut -f1 -d'.' | cut -f1 -d'_')"
|
||||
LANGN=$(grep -E "^${HTSLANG}:" "${DISTPATH}/lang.indexes" | cut -f2 -d':')
|
||||
! test -n "${LANGN}" && LANGN=1
|
||||
|
||||
# Find the browser
|
||||
# note: not all systems have sensible-browser or www-browser alternative
|
||||
|
||||
@@ -54,13 +54,6 @@ refused() {
|
||||
! echo "FAIL: $1 (exit $RC)" || exit 1
|
||||
}
|
||||
|
||||
# assert continue mode was entered: it drops the URL list, so with no cache to
|
||||
# resume the run ends on the usage screen rather than on any other error
|
||||
continued() {
|
||||
{ test "$RC" -ne 0 && grep -q 'usage:' "$1/.log"; } ||
|
||||
! echo "FAIL: $2 (exit $RC)" || exit 1
|
||||
}
|
||||
|
||||
# a value past the old 126/256 caps but within the cap is accepted, on both the
|
||||
# short and long form of each option
|
||||
long=$(nchars 900)
|
||||
@@ -109,50 +102,4 @@ for bad in nan nan:5 5:nan inf 10:5 99999; do
|
||||
refused "#185: invalid --pause '$bad' not refused cleanly"
|
||||
done
|
||||
|
||||
# An option is not -i (continue) merely because its name contains an 'i' (#615).
|
||||
# These used to wipe the URL given before them and exit on the usage screen.
|
||||
run "$tmp/ord-bti" --build-top-index
|
||||
accepted "$tmp/ord-bti" "#615: --build-top-index after the URL wiped the URL list"
|
||||
run "$tmp/ord-bti-s" "-%i"
|
||||
accepted "$tmp/ord-bti-s" "#615: -%i after the URL wiped the URL list"
|
||||
run "$tmp/ord-proto" --protocol 2
|
||||
accepted "$tmp/ord-proto" "#615: --protocol after the URL wiped the URL list"
|
||||
run "$tmp/ord-proto-s" "-@i2"
|
||||
accepted "$tmp/ord-proto-s" "#615: -@i2 after the URL wiped the URL list"
|
||||
|
||||
# %, &, @ and # take the letter after them into another option set, wherever
|
||||
# they sit in the cluster, so the i of -q%i is not the main-set -i either.
|
||||
run "$tmp/ord-qpi" "-q%i"
|
||||
accepted "$tmp/ord-qpi" "#615: -q%i after the URL wiped the URL list"
|
||||
run "$tmp/ord-qai" "-q@i"
|
||||
accepted "$tmp/ord-qai" "#615: -q@i after the URL wiped the URL list"
|
||||
|
||||
# -%q only forces quiet mode, which nothing here can see: a run whose stdout is
|
||||
# not a tty is quiet from the start (htscoremain.c:174). Just check it crawls.
|
||||
run "$tmp/ord-iqs" "-%q"
|
||||
accepted "$tmp/ord-iqs" "#615: -%q after the URL broke the crawl"
|
||||
|
||||
# The real -i still forces continue mode and drops the URL list, in every form:
|
||||
# a fix that merely stopped looking for 'i' would pass the checks above.
|
||||
run "$tmp/cont-s" -i
|
||||
continued "$tmp/cont-s" "#615: -i after the URL no longer forces continue mode"
|
||||
run "$tmp/cont-c" -iC2
|
||||
continued "$tmp/cont-c" "#615: -iC2 after the URL no longer forces continue mode"
|
||||
run "$tmp/cont-l" --continue
|
||||
continued "$tmp/cont-l" "#615: --continue after the URL no longer forces continue mode"
|
||||
run "$tmp/cont-u" --update
|
||||
continued "$tmp/cont-u" "#615: --update after the URL no longer forces continue mode"
|
||||
|
||||
# ...including where another set's letter precedes it in the cluster.
|
||||
run "$tmp/cont-pq" "-%qi"
|
||||
continued "$tmp/cont-pq" "#615: -%qi after the URL no longer forces continue mode"
|
||||
run "$tmp/cont-pm" "-%Mi"
|
||||
continued "$tmp/cont-pm" "#615: -%Mi after the URL no longer forces continue mode"
|
||||
|
||||
# Placed before the URL these always worked; they must keep working.
|
||||
run_only "$tmp/pre-bti" "-%i" "file://$tmp/index.html"
|
||||
accepted "$tmp/pre-bti" "#615: -%i before the URL broke the crawl"
|
||||
run_only "$tmp/pre-proto" "-@i2" "file://$tmp/index.html"
|
||||
accepted "$tmp/pre-proto" "#615: -@i2 before the URL broke the crawl"
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# hts_buildtopindex takes a system-charset path but verif_backblue below it
|
||||
# expects utf-8, mangling a non-ASCII project dir on Windows (#216, #217).
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=topindex "$dir" run |
|
||||
grep -q "topindex self-test OK"
|
||||
@@ -165,13 +165,6 @@ auth="$tmpdir/auth"
|
||||
start_server "$auth" ok
|
||||
run_crawl "$auth/out" "http://${host}:${http_port}/" \
|
||||
"socks5://user:secret@127.0.0.1:${socks_port}"
|
||||
# name a rejected version byte here: the crawl check below only knows the
|
||||
# handshake failed, not why
|
||||
grep -q "^AUTHVER-BAD" "$auth/socks.log" && {
|
||||
echo "FAIL: sub-negotiation version was not RFC 1929's 0x01" >&2
|
||||
cat "$auth/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -rq "ORIGIN-PAGE-563" "$auth/out" || {
|
||||
echo "FAIL: origin not downloaded through the authenticated socks proxy" >&2
|
||||
cat "$auth/out.log" "$auth/socks.log" >&2
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Unit-tests dump_crawl_logs (testlib.sh): the killed test's hts-log.txt must
|
||||
# reach the uploaded logs, or a Windows wedge past --max-time stays
|
||||
# undiagnosable (#605).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/httrack_salv.XXXXXX")
|
||||
export TMPDIR
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
verdict="More than 120 seconds passed.. giving up"
|
||||
|
||||
# Vacuity control: with nothing left behind the assertions below cannot pass on
|
||||
# ambient output.
|
||||
test -z "$(dump_crawl_logs)" || fail "dumped something with no crawl tmpdir"
|
||||
|
||||
# A killed local-crawl.sh leaves exactly this shape.
|
||||
d="${TMPDIR}/httrack_local.probe"
|
||||
mkdir -p "$d/crawl"
|
||||
echo "$verdict" >"$d/crawl/hts-log.txt"
|
||||
echo "httrack stdout marker" >"$d/log"
|
||||
|
||||
# Append to an unterminated line, as a killed test's own log always ends.
|
||||
art="${TMPDIR}/artifact.log"
|
||||
printf '[running httrack ...] ..\t' >"$art"
|
||||
dump_crawl_logs >>"$art"
|
||||
|
||||
grep -qF "$verdict" "$art" || fail "hts-log.txt verdict not salvaged"
|
||||
grep -qF "httrack stdout marker" "$art" || fail "httrack stdout not salvaged"
|
||||
grep -q '^--- .*hts-log.txt' "$art" || fail "dump glued its header onto the unterminated line"
|
||||
|
||||
# Cleared, so the next timing-out test cannot re-report this crawl as its own.
|
||||
test ! -d "$d" || fail "salvaged tmpdir left behind"
|
||||
test -z "$(dump_crawl_logs)" || fail "salvaged crawl reported twice"
|
||||
|
||||
echo "crawl log salvage OK"
|
||||
@@ -1,91 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# webhttrack must map the caller's locale to the right lang.indexes number.
|
||||
# Guards: LC_ALL/LC_MESSAGES were ignored, and zh_tw/pt_br were unreachable.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=$(cd "${testdir}/.." && pwd)
|
||||
script="${distdir}/src/webhttrack"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
test -r "${script}" || fail "no ${script}"
|
||||
test -r "${distdir}/lang.indexes" || fail "no ${distdir}/lang.indexes"
|
||||
|
||||
# Lift the locale code out of the launcher, so this tests the shipped logic itself.
|
||||
funcs=$(sed -n '/^function lang_index/,/^}/p' "${script}")
|
||||
# End on the next section, not a blank line: a blank line grown inside the block
|
||||
# would truncate the lift and silently stop testing everything below it.
|
||||
grep -q '^# Find the browser' "${script}" || fail "locale block terminator moved in ${script}"
|
||||
block=$(sed -n '/^# Locale/,/^# Find the browser/p' "${script}" | sed '$d')
|
||||
echo "${block}" | grep -q 'LANGN=.*lang_index' || fail "could not lift the whole locale block from ${script}"
|
||||
|
||||
# Run the lifted block against whatever locale vars the caller exported.
|
||||
run_block() {
|
||||
# shellcheck disable=SC2034 # read by the eval'd locale block
|
||||
DISTPATH="${distdir}"
|
||||
# the launcher runs under neither -e ("! test -n x && y" returns 1) nor -u
|
||||
set +eu
|
||||
eval "${funcs}"
|
||||
eval "${block}"
|
||||
# shellcheck disable=SC2153 # LANGN is set by the eval'd block, not a LANG typo
|
||||
echo "${LANGN}"
|
||||
}
|
||||
|
||||
# Resolve LANG/LC_ALL/LC_MESSAGES (empty = unset) to the language number.
|
||||
lang_of() {
|
||||
(
|
||||
unset LANG LC_ALL LC_MESSAGES
|
||||
# only the value matters here; hide bash's setlocale gripe when the locale is not installed
|
||||
# shellcheck disable=SC2030 # confining the locale to this subshell is the point
|
||||
{
|
||||
if test -n "$1"; then export LANG="$1"; fi
|
||||
if test -n "$2"; then export LC_ALL="$2"; fi
|
||||
if test -n "$3"; then export LC_MESSAGES="$3"; fi
|
||||
} 2>/dev/null
|
||||
run_block
|
||||
)
|
||||
}
|
||||
|
||||
check() {
|
||||
local want=$1 got desc
|
||||
shift
|
||||
desc="LANG='$1' LC_ALL='$2' LC_MESSAGES='$3'"
|
||||
got=$(lang_of "$1" "$2" "$3")
|
||||
test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
|
||||
echo "ok: ${desc} -> ${got}"
|
||||
}
|
||||
|
||||
# want LANG LC_ALL LC_MESSAGES
|
||||
check 2 "fr_FR.UTF-8" "" ""
|
||||
check 14 "zh_TW.UTF-8" "" "" # region-qualified: Chinese-BIG5, not Simplified (13)
|
||||
check 12 "pt_BR.UTF-8" "" "" # region-qualified: Portugues-Brasil, not Portugues (7)
|
||||
check 13 "zh_CN.UTF-8" "" "" # unlisted region falls back to the bare language
|
||||
check 24 "sl_SI.UTF-8" "" "" # Slovenian is ISO 639-1 "sl"
|
||||
check 4 "" "de_DE.UTF-8" "" # LC_ALL alone
|
||||
check 4 "" "" "de_DE.UTF-8" # LC_MESSAGES alone
|
||||
check 4 "fr_FR.UTF-8" "de_DE.UTF-8" "" # LC_ALL wins over LANG
|
||||
check 4 "fr_FR.UTF-8" "" "de_DE.UTF-8" # LC_MESSAGES wins over LANG
|
||||
check 2 "de_DE.UTF-8" "fr_FR.UTF-8" "de_DE.UTF-8" # LC_ALL wins over LC_MESSAGES
|
||||
check 2 "fr_FR@euro" "" "" # @modifier without a charset
|
||||
check 1 "C" "" "" # C, unknown and unset all mean English
|
||||
check 1 "xx_YY.UTF-8" "" ""
|
||||
check 1 "" "" ""
|
||||
|
||||
# Exported but empty is unset (POSIX), so it must not mask LANG. lang_of cannot
|
||||
# express this: it exports nothing for an empty argument.
|
||||
got=$(
|
||||
unset LANG LC_ALL LC_MESSAGES
|
||||
# shellcheck disable=SC2031 # confining the locale to this subshell is the point
|
||||
{ export LANG="fr_FR.UTF-8" LC_ALL="" LC_MESSAGES=""; } 2>/dev/null
|
||||
run_block
|
||||
)
|
||||
test "${got}" = 2 || fail "empty LC_ALL/LC_MESSAGES must not mask LANG: want 2, got ${got}"
|
||||
echo "ok: LANG='fr_FR.UTF-8' with LC_ALL='' exported -> ${got}"
|
||||
|
||||
echo "PASS"
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# lang/*.txt are line pairs: an English msgid, then its translation. A stray
|
||||
# blank or a drifted msgid silently unhooks every translation that follows,
|
||||
# rather than failing, so assert the pairing and the join against English.txt.
|
||||
# Pairs are physical lines here; a msgid ending in \ continues onto the next one
|
||||
# for the engine (linput_cpp) but not for us. None do today.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
langdir="${top_srcdir:-..}/lang"
|
||||
def="${top_srcdir:-..}/lang.def"
|
||||
eng="$langdir/English.txt"
|
||||
for f in "$eng" "$def"; do
|
||||
[ -f "$f" ] || {
|
||||
echo "cannot find $f"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
keys="$tmp/english.keys"
|
||||
|
||||
# Byte-wise: each file is in its own declared legacy charset, lang.def is CRLF.
|
||||
LC_ALL=C awk 'NR%2==1 { sub(/\r$/, ""); print }' "$eng" >"$keys"
|
||||
|
||||
# An empty parse would make every check below pass vacuously.
|
||||
nkeys=$(wc -l <"$keys")
|
||||
if [ "$nkeys" -lt 400 ]; then
|
||||
echo "only $nkeys msgids parsed from $eng; the parse is broken"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
nfiles=0
|
||||
fail=0
|
||||
for f in "$langdir"/*.txt; do
|
||||
nfiles=$((nfiles + 1))
|
||||
LC_ALL=C awk -v keys="$keys" -v name="${f##*/}" '
|
||||
BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
|
||||
{ sub(/\r$/, "") }
|
||||
NR % 2 == 1 {
|
||||
if ($0 == "")
|
||||
printf "%s:%d: blank line where an English msgid belongs\n", name, NR
|
||||
else if (!($0 in known))
|
||||
printf "%s:%d: msgid absent from English.txt: %s\n", name, NR, $0
|
||||
else
|
||||
next
|
||||
bad++
|
||||
}
|
||||
END {
|
||||
if (NR % 2) {
|
||||
printf "%s: odd line count (%d): msgid/translation pairing is broken\n", name, NR
|
||||
bad++
|
||||
}
|
||||
exit(bad > 0)
|
||||
}
|
||||
' "$f" || fail=1
|
||||
done
|
||||
|
||||
if [ "$nfiles" -lt 25 ]; then
|
||||
echo "only $nfiles language files found in $langdir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# lang.def maps each LANG_* macro to the English string the GUI compiles in,
|
||||
# which is the lookup key into every lang/*.txt: no msgid, no translations.
|
||||
LC_ALL=C awk -v keys="$keys" '
|
||||
BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
|
||||
{ sub(/\r$/, "") }
|
||||
NR % 2 == 1 { macro = $0; next }
|
||||
macro ~ /^(LANG_|LISTDEF_)/ {
|
||||
n++
|
||||
if (!($0 in known)) {
|
||||
printf "lang.def:%d: %s maps to \"%s\", which is not a msgid in English.txt\n", NR, macro, $0
|
||||
bad++
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (n < 400) {
|
||||
printf "lang.def: only %d LANG_ entries parsed; the parse is broken\n", n
|
||||
bad++
|
||||
}
|
||||
exit(bad > 0)
|
||||
}
|
||||
' "$def" || fail=1
|
||||
|
||||
[ "$fail" -eq 0 ] || exit 1
|
||||
|
||||
echo "$nfiles language files consistent with $nkeys English msgids"
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# htsserver's $HOME lookup, through the web UI it actually renders. An exported
|
||||
# but empty $HOME must not resolve the default base path against the root.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
# Mirror local-crawl.sh's skip-with-77: the Debian buildd chroot has no python3.
|
||||
command -v python3 >/dev/null || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
srv=
|
||||
cleanup() {
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
log=$(mktemp)
|
||||
trap 'cleanup; rm -f "${log}"' EXIT
|
||||
|
||||
# Ask htsserver for its rendered default base path, under the given $HOME
|
||||
# ("-" = leave $HOME unset).
|
||||
pathfor() {
|
||||
local homeval=$1 port url got
|
||||
port=$(python3 -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()')
|
||||
|
||||
: >"${log}"
|
||||
(
|
||||
# htsserver ignores SIGTERM/SIGTTOU handling from the harness shell
|
||||
trap '' TERM TTOU
|
||||
if test "${homeval}" = -; then unset HOME; else export HOME="${homeval}"; fi
|
||||
exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
|
||||
) &
|
||||
srv=$!
|
||||
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
|
||||
kill -0 "${srv}" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
|
||||
|
||||
# The UI is served ISO-8859-1, so keep python out of text mode.
|
||||
got=$(python3 -c 'import re, sys, urllib.request
|
||||
body = urllib.request.urlopen(sys.argv[1] + "server/step2.html", timeout=20).read()
|
||||
m = re.search(br"name=\"path\" value=\"([^\"]*)\"", body)
|
||||
print(m.group(1).decode("latin-1") if m else "")' "${url}")
|
||||
|
||||
kill -9 "${srv}" 2>/dev/null || true
|
||||
wait "${srv}" 2>/dev/null || true
|
||||
srv=
|
||||
test -n "${got}" || fail "no path field in the rendered step2.html"
|
||||
echo "${got}"
|
||||
}
|
||||
|
||||
check() {
|
||||
local desc=$1 homeval=$2 want=$3 got
|
||||
got=$(pathfor "${homeval}")
|
||||
test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
|
||||
echo "ok: ${desc} -> ${got}"
|
||||
}
|
||||
|
||||
# A real HOME must reach the page, or the two cases below are vacuous: they
|
||||
# also pass when gethomedir() ignores the environment and always answers ".".
|
||||
check 'a set HOME is used' /nonexistent/hts-home /nonexistent/hts-home/websites
|
||||
|
||||
# Exported but empty is the regression: getenv() returns "", not NULL, so a
|
||||
# bare NULL check let the base path collapse onto /websites.
|
||||
check 'an empty HOME' '' ./websites
|
||||
check 'an unset HOME' - ./websites
|
||||
|
||||
echo "PASS"
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A non-ASCII -O output dir must receive the mirror, not a double-encoded twin
|
||||
# (#621). argv is UTF-8 on Windows, so path_html was re-encoded as if it were
|
||||
# the ANSI codepage, prefixing every saved file with a mangled directory. Only
|
||||
# path_html is the non-ASCII "café"; path_log stays ASCII so the harness reads
|
||||
# hts-log.txt/hts-cache normally and this isolates the -O encoding path.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --html-subdir 'café' --errors 0 --files 5 \
|
||||
--found 'simple/basic.html' \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
@@ -71,7 +71,6 @@ TESTS = \
|
||||
01_engine-status.test \
|
||||
01_engine-stripquery.test \
|
||||
01_engine-strsafe.test \
|
||||
01_engine-topindex.test \
|
||||
01_engine-urlhack.test \
|
||||
01_engine-unescape-bounds.test \
|
||||
01_engine-useragent.test \
|
||||
@@ -140,11 +139,6 @@ TESTS = \
|
||||
56_local-proxy-noleak.test \
|
||||
57_local-proxy-connect.test \
|
||||
58_watchdog.test \
|
||||
59_local-tls-stall.test \
|
||||
60_crawl-log-salvage.test \
|
||||
61_webhttrack-locale.test \
|
||||
62_lang-integrity.test \
|
||||
63_webhttrack-home.test \
|
||||
64_local-intl-outdir.test
|
||||
59_local-tls-stall.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -47,7 +47,6 @@ key="${testdir}/server.key"
|
||||
|
||||
tls=
|
||||
verbose=
|
||||
html_subdir=
|
||||
rerun=
|
||||
rerun_args=
|
||||
rerun_dead=
|
||||
@@ -133,13 +132,6 @@ while test "$pos" -lt "$nargs"; do
|
||||
pos=$((pos + 1))
|
||||
rerun_args="${args[$pos]}"
|
||||
;;
|
||||
--html-subdir)
|
||||
# Mirror into "$out/NAME" (path_html) but keep logs/cache in "$out"
|
||||
# (path_log): a non-ASCII NAME exercises the -O path encoding (#621)
|
||||
# without routing the harness-read hts-log.txt through a non-ASCII path.
|
||||
pos=$((pos + 1))
|
||||
html_subdir="${args[$pos]}"
|
||||
;;
|
||||
--errors | --errors-content | --files)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
@@ -215,21 +207,12 @@ test -n "$ver" || die "could not run httrack"
|
||||
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out" || die "could not create $out"
|
||||
# path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
|
||||
# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME"
|
||||
# (the mirror root the audits inspect) while path_log stays "$out".
|
||||
mirrorroot="$out"
|
||||
odir="$out"
|
||||
if test -n "$html_subdir"; then
|
||||
mirrorroot="${out}/${html_subdir}"
|
||||
odir="${mirrorroot},${out}"
|
||||
fi
|
||||
# Localhost is fast; disable the rate/bandwidth safety limits but keep a
|
||||
# max-time backstop so a hang cannot wedge the suite.
|
||||
declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
|
||||
log="${tmpdir}/log"
|
||||
info "running httrack ${hts[*]}"
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
@@ -246,7 +229,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
|
||||
# --- optional second pass: re-mirror into the same dir (cache/update path) ----
|
||||
if test -n "$rerun"; then
|
||||
info "re-running httrack (update pass)"
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
@@ -274,7 +257,7 @@ fi
|
||||
if test -n "$rerun_args"; then
|
||||
read -ra extra <<<"$rerun_args"
|
||||
info "re-running httrack with ${rerun_args}"
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
@@ -296,7 +279,7 @@ if test -n "$rerun_dead"; then
|
||||
stop_server "$serverpid"
|
||||
serverpid=
|
||||
info "re-running httrack against the stopped server"
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid" || true
|
||||
@@ -325,7 +308,7 @@ fi
|
||||
|
||||
# --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
|
||||
hostroot=
|
||||
for cand in "${mirrorroot}/127.0.0.1_${port}" "${mirrorroot}/127.0.0.1"; do
|
||||
for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
|
||||
if test -d "$cand"; then
|
||||
hostroot="$cand"
|
||||
break
|
||||
|
||||
@@ -24,7 +24,6 @@ from proxytestlib import bind_ephemeral, pipe # noqa: E402
|
||||
# The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
|
||||
# so a locally-resolving client could not reach us -- success proves remote DNS.
|
||||
REMOTE_HOST = b"socks-origin.invalid"
|
||||
AUTH_VERSION = 0x01 # RFC 1929 sub-negotiation version
|
||||
# index links the subpages so an -r3 crawl reuses one keep-alive socket
|
||||
LINKS = "".join('<a href="/p%d.html">%d</a>' % (i, i) for i in range(1, 6))
|
||||
ORIGIN_BODY = ("<html><body>ORIGIN-PAGE-563 " + LINKS + "</body></html>").encode()
|
||||
@@ -105,12 +104,6 @@ def negotiate_auth(conn, logdir):
|
||||
if 0x02 in methods: # prefer user/pass so the auth test exercises RFC 1929
|
||||
conn.sendall(b"\x05\x02")
|
||||
(subver,) = recvn(conn, 1)
|
||||
# RFC 1929's version byte is 0x01, not SOCKS5's 0x05: a real proxy
|
||||
# rejects a mismatch instead of tunnelling anyway
|
||||
if subver != AUTH_VERSION:
|
||||
log(logdir, "AUTHVER-BAD %d" % subver)
|
||||
conn.sendall(bytes([AUTH_VERSION, 0x01])) # sub-negotiation failure
|
||||
return False
|
||||
(ulen,) = recvn(conn, 1)
|
||||
uname = recvn(conn, ulen)
|
||||
(plen,) = recvn(conn, 1)
|
||||
|
||||
@@ -44,25 +44,6 @@ stop_server() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Dump and clear the crawl logs a hard-killed test leaves in TMPDIR (its cleanup
|
||||
# trap never ran): hts-log.txt alone records "More than N seconds passed.. giving
|
||||
# up", so a wedge past --max-time is undiagnosable without it (#605).
|
||||
dump_crawl_logs() {
|
||||
local d f
|
||||
for d in "${TMPDIR:-/tmp}"/httrack_local.*; do
|
||||
test -d "$d" || continue
|
||||
for f in "$d/crawl/hts-log.txt" "$d/log" "$d/log.2"; do
|
||||
test -f "$f" || continue
|
||||
# Leading newline: the killed test's last line has no terminator.
|
||||
printf '\n--- %s (last 200 lines)\n' "$f"
|
||||
tail -n 200 "$f"
|
||||
done
|
||||
# so a later test's dump cannot re-report this one; never fatal, the
|
||||
# caller is already handling a failure and Windows may still hold a file
|
||||
rm -rf "$d" || true
|
||||
done
|
||||
}
|
||||
|
||||
# Kill a backgrounded job and its whole descendant tree. POSIX: the caller must
|
||||
# have put the job in its own process group (run_with_timeout does) so we signal
|
||||
# the group; a bare kill would orphan the grandchildren. Windows: the tree is
|
||||
|
||||
Reference in New Issue
Block a user