mirror of
https://github.com/xroche/httrack.git
synced 2026-07-27 19:12:54 +03:00
Compare commits
8 Commits
fix/strlen
...
fix/failed
| Author | SHA1 | Date | |
|---|---|---|---|
|
3563d4fbe0 | ||
|
|
45630f63b4 | ||
|
|
2ffe8d5582 | ||
|
|
14cc7a13fc | ||
|
|
1a7effce49 | ||
|
|
6457828b72 | ||
|
|
f118cd86d3 | ||
|
|
1373530f17 |
18
.github/workflows/ci.yml
vendored
18
.github/workflows/ci.yml
vendored
@@ -79,10 +79,7 @@ jobs:
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
# A backstop only: tests/test-timeout.sh bounds each test, and a healthy run
|
||||
# is a minute here, two on macOS. Without it a stall ran to the job's 6h default.
|
||||
- name: Test
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
@@ -122,7 +119,6 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test without python3
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Hide every python3* so `command -v python3` fails like it does in the
|
||||
@@ -177,7 +173,6 @@ jobs:
|
||||
sudo ifconfig lo0 alias 127.0.0.3 up
|
||||
|
||||
- name: Test
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
@@ -250,7 +245,6 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
@@ -308,7 +302,6 @@ jobs:
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
@@ -352,7 +345,6 @@ jobs:
|
||||
- name: Test (offline self-tests under MSan)
|
||||
env:
|
||||
MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
|
||||
@@ -434,7 +426,6 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
@@ -627,12 +618,7 @@ jobs:
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
|
||||
# Merge base, not the branch tip: master moves during a run, and a
|
||||
# tip-relative diff blames its new C on this PR.
|
||||
if ! base="$(git merge-base "origin/${{ github.base_ref }}" HEAD)"; then
|
||||
echo "::error::no merge base with origin/${{ github.base_ref }}; cannot scope the check."
|
||||
exit 1
|
||||
fi
|
||||
base="origin/${{ github.base_ref }}"
|
||||
set +e
|
||||
diff="$(git clang-format --binary clang-format-19 --style=file \
|
||||
--diff --extensions c,h "$base")"
|
||||
@@ -674,8 +660,6 @@ jobs:
|
||||
git fetch --no-tags origin \
|
||||
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
|
||||
base="origin/${{ github.base_ref }}"
|
||||
# Three dots: merge-base scoped, so commits landing on master mid-run
|
||||
# are not counted as this PR's.
|
||||
changed="$(git diff --name-only "$base"...HEAD)"
|
||||
has() { printf '%s\n' "$changed" | grep -qx "$1"; }
|
||||
if has man/httrack.1 && ! has html/httrack.man.html; then
|
||||
|
||||
58
.github/workflows/windows-build.yml
vendored
58
.github/workflows/windows-build.yml
vendored
@@ -189,53 +189,25 @@ jobs:
|
||||
# A wedged crawl must not eat the job's timeout budget. timeout(1)'s
|
||||
# signals can't reap a native httrack.exe (MSYS signals don't reach it),
|
||||
# so a hang orphaned processes that starved the runner; run_with_timeout
|
||||
# TerminateProcess-es the whole tree. 600s is unchanged: it clears the
|
||||
# 540s a three-pass crawl may legitimately take under local-crawl.sh's
|
||||
# own watchdogs, against a slowest healthy test here of 39s.
|
||||
# TerminateProcess-es the whole tree. 600s clears the slowest multi-pass
|
||||
# crawl (a few passes at --max-time=120 each).
|
||||
. ./testlib.sh
|
||||
per_test=600
|
||||
|
||||
# The whole suite must give up before the step timeout above. A cancelled
|
||||
# step keeps neither its log nor the artifacts the later if:always()
|
||||
# steps would upload, so an overrun that ends in a cancel tells us
|
||||
# nothing; failing on our own terms keeps both. Healthy runs take 8-9
|
||||
# min. The check sits between tests, so the step can still reach 25 min
|
||||
# plus one per-test budget, and that worst case stays inside the 45.
|
||||
suite_deadline=1500
|
||||
started=$SECONDS
|
||||
|
||||
# Survives into the artifact even if the tail of the step log does not.
|
||||
progress=suite-progress.log
|
||||
: >"$progress"
|
||||
|
||||
pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
|
||||
pass=0 fail=0 skip=0 failed="" skipped=""
|
||||
for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
|
||||
*_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
|
||||
60_crawl-log-salvage.test 116_engine-rtrim.test; do
|
||||
elapsed=$((SECONDS - started))
|
||||
if [ "$elapsed" -ge "$suite_deadline" ]; then
|
||||
echo "::error::suite deadline: ${elapsed}s elapsed, stopping before $t"
|
||||
echo "DEADLINE before $t after ${elapsed}s" >>"$progress"
|
||||
# Per-test start times, so the slow ones are named rather than guessed.
|
||||
sed 's/^/ /' "$progress"
|
||||
deadline=1
|
||||
break
|
||||
fi
|
||||
echo "RUN $t at ${elapsed}s" >>"$progress"
|
||||
60_crawl-log-salvage.test; do
|
||||
rc=0
|
||||
# Same guard "make check" uses on POSIX, so a wedge is diagnosed the
|
||||
# same way on every platform. It dumps before it kills, which a bare
|
||||
# run_with_timeout cannot: by the time that returns, the tree whose
|
||||
# stack we wanted is already gone.
|
||||
HTTRACK_TEST_TIMEOUT=$per_test bash ./test-timeout.sh "$t" >"$t.log" 2>&1 || rc=$?
|
||||
run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
case "$rc" in
|
||||
0) pass=$((pass + 1)); echo "PASS $t" ;;
|
||||
77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
|
||||
124)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
# test-timeout.sh has already written the process list, the stacks
|
||||
# and the killed crawl's own logs into $t.log.
|
||||
echo "FAIL $t (timed out, tree killed)"
|
||||
# Re-running a wedge traced would just hang again: salvage the
|
||||
# killed crawl's own logs into the artifact instead.
|
||||
dump_crawl_logs >>"$t.log"
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
*)
|
||||
@@ -243,29 +215,21 @@ jobs:
|
||||
echo "FAIL $t (exit $rc)"
|
||||
# These assert with `test "$(...)" == "..." || exit 1`, which
|
||||
# says nothing at all on failure. Re-run traced, still bounded.
|
||||
run_with_timeout "$per_test" bash -x "$t" >>"$t.log" 2>&1 || true
|
||||
run_with_timeout 600 bash -x "$t" >>"$t.log" 2>&1 || true
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
esac
|
||||
echo "$rc $t" >>"$progress"
|
||||
# An orphaned native httrack.exe spins and starves the runner, which
|
||||
# is how this job dies with "lost communication" rather than a plain
|
||||
# timeout. Clear them between tests and name whoever leaked them.
|
||||
reap_leftover_processes "$t" | tee -a "$progress"
|
||||
done
|
||||
echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
|
||||
tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
# footer-overflow and purge-longpath skip on Windows (need a path past MAX_PATH);
|
||||
# crange pending #581;
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
|
||||
# webdav-mime needs a reapable background listener, which MSYS cannot give it;
|
||||
# badmtime needs a filesystem that stores an mtime past gmtime's range;
|
||||
# single-file ends on a GUI half needing htsserver, which this job does not build.
|
||||
expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
|
||||
# First, or the deadline reads as an unexplained shortfall in the gates below.
|
||||
[ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
@@ -26,10 +26,6 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
check`, or `PATH="<bld>/src:$PATH"` for a manual run.
|
||||
- Give new `.test` scripts `set -e`: the older ones predate the rule, so several
|
||||
`local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
|
||||
- Run teardown with errexit off: `trap 'set +e; cleanup' EXIT`. Under `set -e` a
|
||||
failing cleanup command becomes the test's exit status (#773). Keep the other
|
||||
signals on their own `trap` line, or errexit stays off for the rest of the run.
|
||||
The guard also resets `$?`, so save it first if teardown reads it.
|
||||
- Never assert with `cmd | grep -q MARKER && fail`. Under `pipefail` the
|
||||
pipeline is non-zero both when `cmd` fails and when `grep -q` matches early
|
||||
and SIGPIPEs it, so the `&&` never fires and a probe that proved nothing reads
|
||||
|
||||
@@ -487,6 +487,141 @@ regen:
|
||||
|
||||
#define HTS_DATA_UNKNOWN_HTML_LEN 0
|
||||
|
||||
#define HTS_DATA_ERROR_HTML "<html>"LF\
|
||||
"<html xmlns=\"http://www.w3.org/1999/xhtml\" lang=\"en\">"LF\
|
||||
""LF\
|
||||
"<head>"LF\
|
||||
" <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />"LF\
|
||||
" <meta name=\"description\" content=\"HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help\" />"LF\
|
||||
" <meta name=\"keywords\" content=\"httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software\" />"LF\
|
||||
" <title>Page not retrieved! - HTTrack Website Copier</title>"LF\
|
||||
" <style type=\"text/css\">"LF\
|
||||
" <!--"LF\
|
||||
""LF\
|
||||
"body {"LF\
|
||||
" margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;"LF\
|
||||
" background: #77b;"LF\
|
||||
"}"LF\
|
||||
"body, td {"LF\
|
||||
" font: 14px \"Trebuchet MS\", Verdana, Arial, Helvetica, sans-serif;"LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
"#subTitle {"LF\
|
||||
" background: #000; color: #fff; padding: 4px; font-weight: bold; "LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
"#siteNavigation a, #siteNavigation .current {"LF\
|
||||
" font-weight: bold; color: #448;"LF\
|
||||
" }"LF\
|
||||
"#siteNavigation a:link { text-decoration: none; }"LF\
|
||||
"#siteNavigation a:visited { text-decoration: none; }"LF\
|
||||
""LF\
|
||||
"#siteNavigation .current { background-color: #ccd; }"LF\
|
||||
""LF\
|
||||
"#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }"LF\
|
||||
"#siteNavigation a:active { text-decoration: none; background-color: #ccc; }"LF\
|
||||
""LF\
|
||||
""LF\
|
||||
"a:link { text-decoration: underline; color: #00f; }"LF\
|
||||
"a:visited { text-decoration: underline; color: #000; }"LF\
|
||||
"a:hover { text-decoration: underline; color: #c00; }"LF\
|
||||
"a:active { text-decoration: underline; }"LF\
|
||||
""LF\
|
||||
"#pageContent {"LF\
|
||||
" clear: both;"LF\
|
||||
" border-bottom: 6px solid #000;"LF\
|
||||
" padding: 10px; padding-top: 20px;"LF\
|
||||
" line-height: 1.65em;"LF\
|
||||
" background-image: url(backblue.gif);"LF\
|
||||
" background-repeat: no-repeat;"LF\
|
||||
" background-position: top right;"LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
"#pageContent, #siteNavigation {"LF\
|
||||
" background-color: #ccd;"LF\
|
||||
" }"LF\
|
||||
""LF\
|
||||
""LF\
|
||||
".imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }"LF\
|
||||
".imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }"LF\
|
||||
""LF\
|
||||
"hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }"LF\
|
||||
""LF\
|
||||
"h1 { margin: 0; font-weight: bold; font-size: 2em; }"LF\
|
||||
"h2 { margin: 0; font-weight: bold; font-size: 1.6em; }"LF\
|
||||
"h3 { margin: 0; font-weight: bold; font-size: 1.3em; }"LF\
|
||||
"h4 { margin: 0; font-weight: bold; font-size: 1.18em; }"LF\
|
||||
""LF\
|
||||
".blak { background-color: #000; }"LF\
|
||||
".hide { display: none; }"LF\
|
||||
".tableWidth { min-width: 400px; }"LF\
|
||||
""LF\
|
||||
".tblRegular { border-collapse: collapse; }"LF\
|
||||
".tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }"LF\
|
||||
".tblHeaderColor, .tblHeaderColor td { background: #99c; }"LF\
|
||||
".tblNoBorder td { border: 0; }"LF\
|
||||
""LF\
|
||||
""LF\
|
||||
"// -->"LF\
|
||||
"</style>"LF\
|
||||
""LF\
|
||||
"</head>"LF\
|
||||
""LF\
|
||||
"<table width=\"76%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"3\" class=\"tableWidth\">"LF\
|
||||
" <tr>"LF\
|
||||
" <td id=\"subTitle\">HTTrack Website Copier - Open Source offline browser</td>"LF\
|
||||
" </tr>"LF\
|
||||
"</table>"LF\
|
||||
"<table width=\"76%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"0\" class=\"tableWidth\">"LF\
|
||||
"<tr class=\"blak\">"LF\
|
||||
"<td>"LF\
|
||||
" <table width=\"100%%\" border=\"0\" align=\"center\" cellspacing=\"1\" cellpadding=\"0\">"LF\
|
||||
" <tr>"LF\
|
||||
" <td colspan=\"6\"> "LF\
|
||||
" <table width=\"100%%\" border=\"0\" align=\"center\" cellspacing=\"0\" cellpadding=\"10\">"LF\
|
||||
" <tr> "LF\
|
||||
" <td id=\"pageContent\"> "LF\
|
||||
"<!-- ==================== End prologue ==================== -->"LF\
|
||||
"<h1><strong><u>Oops!...</u></strong></h1>"LF\
|
||||
"<h3>This page has <font color=\"red\"><em>not</em></font> been retrieved by HTTrack Website Copier (%s). </h3>"LF\
|
||||
"<script language=\"Javascript\">"LF\
|
||||
"<!--"LF\
|
||||
" var loc=document.location.toString();"LF\
|
||||
" if (loc) {"LF\
|
||||
" var pos=loc.indexOf('link=');"LF\
|
||||
" if (pos>0) {"LF\
|
||||
" document.write('Clic to the link <b>below</b> to go to the online location!<br><a href=\"'+loc.substring(pos+5)+'\">'+loc.substring(pos+5)+'</a><br>');"LF\
|
||||
" } else"LF\
|
||||
" document.write('(no location defined)');"LF\
|
||||
" }"LF\
|
||||
"// -->"LF\
|
||||
"</script>"LF\
|
||||
"<h6 align=\"right\">Mirror by HTTrack Website Copier</h6>"LF\
|
||||
"</body>"LF\
|
||||
"</html>"LF\
|
||||
"<!-- ==================== Start epilogue ==================== -->"LF\
|
||||
" </td>"LF\
|
||||
" </tr>"LF\
|
||||
" </table>"LF\
|
||||
" </td>"LF\
|
||||
" </tr>"LF\
|
||||
" </table>"LF\
|
||||
"</td>"LF\
|
||||
"</tr>"LF\
|
||||
"</table>"LF\
|
||||
""LF\
|
||||
"<table width=\"76%%\" height=\"100%%\" border=\"0\" align=\"center\" valign=\"bottom\" cellspacing=\"0\" cellpadding=\"0\">"LF\
|
||||
" <tr>"LF\
|
||||
" <td id=\"footer\"><small>© 2014 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\
|
||||
" </tr>"LF\
|
||||
"</table>"LF\
|
||||
""LF\
|
||||
"</body>"LF\
|
||||
""LF\
|
||||
"</html>"LF\
|
||||
""LF\
|
||||
""LF
|
||||
|
||||
// image gif "unknown"
|
||||
#define HTS_DATA_UNKNOWN_GIF \
|
||||
"\x47\x49\x46\x38\x39\x61\x20\x0\x20\x0\xf7\xff\x0\xc0\xc0\xc0\xff\x0\x0\xfc\x3\x0\xf8\x6\x0\xf6\x9\x0\xf2\xc\x0\xf0\xf\x0\xf0\xe\x0\xed\x11\x0\xec\x13\x0\xeb\x14\x0\xe9\x15\x0\xe8\x18\x0\xe6\x18\x0\xe5\x1a\x0\xe3\x1c\x0\xe2\x1d\x0\xe1\x1e\x0\xdf\x20\x0\xdd\x23\x0\xdd\x22\x0\xdb\x23\x0\xda\x25\x0\xd9\x25\x0\xd8\x27\x0\xd6\x29\x0\xd5\x2a\x0\xd3\x2c\x0\xd2\x2d\x0"\
|
||||
|
||||
@@ -501,11 +501,13 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
|
||||
/* read line */
|
||||
linput(fp, line, 250);
|
||||
hts_lowcase(line);
|
||||
/* trim first: a blank line is skipped, not parsed as an option */
|
||||
hts_rtrim(line, HTS_REALSPACES);
|
||||
if (strnotempty(line)) {
|
||||
/* no comment line: # // ; */
|
||||
if (strchr("#/;", line[0]) == NULL) {
|
||||
/* right trim */
|
||||
a = line + strlen(line) - 1;
|
||||
while(is_realspace(*a))
|
||||
*(a--) = '\0';
|
||||
/* jump "set " and spaces */
|
||||
a = line;
|
||||
while(is_realspace(*a))
|
||||
@@ -575,6 +577,7 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
|
||||
@@ -340,61 +340,12 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr
|
||||
}
|
||||
|
||||
static int slot_can_be_cached_on_disk(const lien_back * back) {
|
||||
/* A pending backup or spool means the slot is not finalized, and the swap
|
||||
would unlink it through back_clear_entry() (#771). */
|
||||
if (back->tmpfile != NULL && back->tmpfile[0] != '\0')
|
||||
return 0;
|
||||
return (back->status == STATUS_READY && back->locked == 0
|
||||
&& back->url_sav[0] != '\0'
|
||||
&& strcmp(back->url_sav, BACK_ADD_TEST) != 0);
|
||||
/* Note: not checking !IS_DELAYED_EXT(back->url_sav) or it will quickly cause the slots to be filled! */
|
||||
}
|
||||
|
||||
int back_selftest_slot_swap(void) {
|
||||
lien_back back;
|
||||
int err = 0;
|
||||
|
||||
#define CHECK(want, why) \
|
||||
do { \
|
||||
if (slot_can_be_cached_on_disk(&back) != (want)) { \
|
||||
fprintf(stderr, "backswap: expected %d for %s\n", (want), (why)); \
|
||||
err = 1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
memset(&back, 0, sizeof(back));
|
||||
back.status = STATUS_READY;
|
||||
strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
|
||||
CHECK(1, "a plain ready slot");
|
||||
|
||||
back.tmpfile = back.tmpfile_buffer;
|
||||
strcpybuff(back.tmpfile_buffer, "/tmp/httrack-selftest.bin.bak");
|
||||
CHECK(0, "a slot still holding a re-fetch backup");
|
||||
|
||||
/* Callers clear a spent temporary by emptying the name, not the pointer. */
|
||||
back.tmpfile_buffer[0] = '\0';
|
||||
CHECK(1, "a slot whose temporary was already dropped");
|
||||
|
||||
back.tmpfile = NULL;
|
||||
back.locked = 1;
|
||||
CHECK(0, "a locked slot");
|
||||
back.locked = 0;
|
||||
|
||||
back.status = STATUS_TRANSFER;
|
||||
CHECK(0, "a slot still transferring");
|
||||
back.status = STATUS_READY;
|
||||
|
||||
back.url_sav[0] = '\0';
|
||||
CHECK(0, "a slot with no save name");
|
||||
|
||||
strcpybuff(back.url_sav, BACK_ADD_TEST);
|
||||
CHECK(0, "the dummy test slot");
|
||||
#undef CHECK
|
||||
|
||||
printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Put all backing entries that are ready in the storage hashtable to spare space and CPU */
|
||||
int back_cleanup_background(httrackp * opt, cache_back * cache,
|
||||
struct_back * sback) {
|
||||
@@ -634,29 +585,6 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Note: utf-8 */
|
||||
void back_refetch_backup(httrackp *opt, lien_back *const back) {
|
||||
back->tmpfile = NULL;
|
||||
if (fexist_utf8(back->url_sav)) {
|
||||
hts_boolean saved = HTS_FALSE;
|
||||
|
||||
if (create_back_tmpfile(opt, back, "bak") == 0) {
|
||||
/* clobber a .bak a killed run left behind, or the guard stays off for
|
||||
good (#758) */
|
||||
if (fexist_utf8(back->tmpfile))
|
||||
hts_log_print(opt, LOG_WARNING, "replacing leftover backup %s",
|
||||
back->tmpfile);
|
||||
saved = hts_rename_over(back->url_sav, back->tmpfile);
|
||||
}
|
||||
if (!saved) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"could not back up %s; an aborted re-fetch will lose it",
|
||||
back->url_sav);
|
||||
back->tmpfile = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Did the fetch fail to produce a response, as opposed to the engine
|
||||
deliberately passing the resource over? Only the latter may be purged. */
|
||||
static hts_boolean back_transfer_failed(const int statuscode) {
|
||||
@@ -3236,7 +3164,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].url_sav, 1, 1,
|
||||
back[i].r.notmodified);
|
||||
back[i].r.compressed = 0;
|
||||
back_refetch_backup(opt, &back[i]);
|
||||
/* Re-fetch over an existing file (#77 follow-up):
|
||||
move the good copy aside before truncating it
|
||||
so an aborted transfer can restore it. url_sav
|
||||
is still written normally (file list intact).
|
||||
*/
|
||||
back[i].tmpfile = NULL;
|
||||
if (fexist_utf8(back[i].url_sav)) {
|
||||
if (create_back_tmpfile(opt, &back[i], "bak") !=
|
||||
0 ||
|
||||
RENAME(back[i].url_sav, back[i].tmpfile) !=
|
||||
0) {
|
||||
back[i].tmpfile = NULL;
|
||||
}
|
||||
}
|
||||
if ((back[i].r.out =
|
||||
filecreate(&opt->state.strc,
|
||||
back[i].url_sav)) == NULL) {
|
||||
|
||||
@@ -139,12 +139,6 @@ int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p);
|
||||
int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p);
|
||||
/* Move the previous copy of back->url_sav to back->tmpfile so back_finalize()
|
||||
can put it back when the re-fetch fails (#77 follow-up). Call right before
|
||||
truncating url_sav; tmpfile stays NULL when there is nothing to save. */
|
||||
void back_refetch_backup(httrackp *opt, lien_back *const back);
|
||||
/* -#test=backswap: slots eligible for the on-disk ready table. */
|
||||
int back_selftest_slot_swap(void);
|
||||
void back_info(struct_back * sback, int i, int j, FILE * fp);
|
||||
void back_infostr(struct_back *sback, int i, int j, char *s, size_t size);
|
||||
LLint back_transferred(LLint add, struct_back * sback);
|
||||
|
||||
@@ -491,7 +491,7 @@ char *bauth_prefix(char *prefix, const char *adr, const char *fil) {
|
||||
if (a)
|
||||
*a = '\0';
|
||||
if (strchr(prefix, '/')) {
|
||||
a = hts_lastcharptr(prefix);
|
||||
a = prefix + strlen(prefix) - 1;
|
||||
while(*a != '/')
|
||||
a--;
|
||||
*(a + 1) = '\0';
|
||||
|
||||
@@ -855,6 +855,13 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
return r;
|
||||
}
|
||||
|
||||
// lecture d'un fichier dans le cache
|
||||
// si save==null alors test unqiquement
|
||||
static int hts_rename(httrackp * opt, const char *a, const char *b) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: rename %s -> %s (%p %p)", a, b, a, b);
|
||||
return RENAME(a, b);
|
||||
}
|
||||
|
||||
/* Open the cache ZIP via hts_fopen_utf8 so a non-ASCII path_log isn't mangled
|
||||
to ANSI (#630); 64-bit funcs keep multi-GB caches whole on Windows LLP64. */
|
||||
static voidpf ZCALLBACK hts_zip_fopen_utf8(voidpf opaque, const void *filename,
|
||||
@@ -984,15 +991,29 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))) { // a previous cache exists.. rename it
|
||||
if (!hts_rename_over(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"),
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip"))) {
|
||||
/* Remove OLD cache */
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")) !=
|
||||
0) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"Cache: error while moving previous cache");
|
||||
}
|
||||
}
|
||||
|
||||
/* Rename */
|
||||
if (hts_rename
|
||||
(opt,
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) != 0) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"Cache: error while moving previous cache");
|
||||
} else {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: rotated new.zip to old.zip");
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -676,7 +676,7 @@ int cache_selftests(httrackp *opt, const char *dir) {
|
||||
char base[HTS_URLMAXSIZE];
|
||||
|
||||
strcpybuff(base, dir);
|
||||
if (base[0] != '\0' && hts_lastchar(base) != '/') {
|
||||
if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
|
||||
strcatbuff(base, "/");
|
||||
}
|
||||
StringCopy(opt->path_log, base);
|
||||
@@ -856,7 +856,7 @@ static void golden_setup(httrackp *opt, const char *dir) {
|
||||
char base[HTS_URLMAXSIZE];
|
||||
|
||||
strcpybuff(base, dir);
|
||||
if (base[0] != '\0' && hts_lastchar(base) != '/') {
|
||||
if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
|
||||
strcatbuff(base, "/");
|
||||
}
|
||||
StringCopy(opt->path_log, base);
|
||||
|
||||
@@ -773,7 +773,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// sauter les + sans rien après..
|
||||
if (strnotempty(tempo)) {
|
||||
if ((plus == 0) && (type == 1)) { // implicite: *www.edf.fr par exemple
|
||||
if (hts_lastchar(tempo) != '*') {
|
||||
if (tempo[strlen(tempo) - 1] != '*') {
|
||||
strcatbuff(tempo, "*"); // ajouter un *
|
||||
}
|
||||
}
|
||||
@@ -1819,6 +1819,60 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
// -- -- --
|
||||
// sauver fichier
|
||||
|
||||
/* En cas d'erreur, vérifier que fichier d'erreur existe */
|
||||
if (strnotempty(savename()) == 0) { // chemin de sauvegarde existant
|
||||
if (strcmp(urlfil(), "/robots.txt") == 0) { // pas robots.txt
|
||||
if (store_errpage) { // c'est une page d'erreur
|
||||
int create_html_warning = 0;
|
||||
int create_gif_warning = 0;
|
||||
|
||||
switch (ishtml(opt, urlfil())) { /* pas fichier html */
|
||||
case 0: /* non html */
|
||||
{
|
||||
char buff[256];
|
||||
|
||||
guess_httptype_sized(opt, buff, sizeof(buff), urlfil());
|
||||
if (strcmp(buff, "image/gif") == 0)
|
||||
create_gif_warning = 1;
|
||||
}
|
||||
break;
|
||||
case 1: /* html */
|
||||
if (!r.adr) {
|
||||
}
|
||||
break;
|
||||
default: /* don't know.. */
|
||||
break;
|
||||
}
|
||||
/* Créer message d'erreur ? */
|
||||
if (create_html_warning) {
|
||||
char *adr =
|
||||
(char *) malloct(strlen(HTS_DATA_ERROR_HTML) + 1100);
|
||||
hts_log_print(opt, LOG_INFO, "Creating HTML warning file (%s)",
|
||||
r.msg);
|
||||
if (adr) {
|
||||
if (r.adr) {
|
||||
freet(r.adr);
|
||||
r.adr = NULL;
|
||||
}
|
||||
sprintf(adr, HTS_DATA_ERROR_HTML, r.msg);
|
||||
r.adr = adr;
|
||||
}
|
||||
} else if (create_gif_warning) {
|
||||
char *adr = (char *) malloct(HTS_DATA_UNKNOWN_GIF_LEN);
|
||||
|
||||
hts_log_print(opt, LOG_INFO, "Creating GIF dummy file (%s)",
|
||||
r.msg);
|
||||
if (r.adr) {
|
||||
freet(r.adr);
|
||||
r.adr = NULL;
|
||||
}
|
||||
memcpy(adr, HTS_DATA_UNKNOWN_GIF, HTS_DATA_UNKNOWN_GIF_LEN);
|
||||
r.adr = adr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (strnotempty(savename()) == 0) { // pas de chemin de sauvegarde
|
||||
if (strcmp(urlfil(), "/robots.txt") == 0) { // robots.txt
|
||||
char BIGSTK sitemaps[8192];
|
||||
@@ -2136,9 +2190,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
continue;
|
||||
strcpybuff(file, StringBuff(opt->path_html));
|
||||
strcatbuff(file, line + 1);
|
||||
/* strip filenote()'s ']', absent when linput() truncated the
|
||||
line */
|
||||
hts_striplastchar(file, ']');
|
||||
file[strlen(file) - 1] = '\0';
|
||||
hts_changes_previous(opt, file + StringLength(opt->path_html));
|
||||
if (!strstr(adr, line)) { // not found in the new list?
|
||||
if (fexist_utf8(file)) { // still on disk
|
||||
@@ -2165,11 +2217,12 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
fseek(old_lst, 0, SEEK_SET);
|
||||
while(!feof(old_lst)) {
|
||||
linput(old_lst, line, 1000);
|
||||
while (strnotempty(line) && (hts_lastchar(line) != '/') &&
|
||||
(hts_lastchar(line) != '\\')) {
|
||||
hts_choplastchar(line);
|
||||
while(strnotempty(line) && (line[strlen(line) - 1] != '/')
|
||||
&& (line[strlen(line) - 1] != '\\')) {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
}
|
||||
hts_choplastchar(line);
|
||||
if (strnotempty(line))
|
||||
line[strlen(line) - 1] = '\0';
|
||||
if (strnotempty(line))
|
||||
if (!strstr(adr, line)) { // non trouvé?
|
||||
char BIGSTK file[HTS_URLMAXSIZE * 2];
|
||||
@@ -2182,12 +2235,13 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
if (opt->log) {
|
||||
hts_log_print(opt, LOG_INFO, "Purging directory %s/",
|
||||
file);
|
||||
while (strnotempty(file) &&
|
||||
(hts_lastchar(file) != '/') &&
|
||||
(hts_lastchar(file) != '\\')) {
|
||||
hts_choplastchar(file);
|
||||
while(strnotempty(file)
|
||||
&& (file[strlen(file) - 1] != '/')
|
||||
&& (file[strlen(file) - 1] != '\\')) {
|
||||
file[strlen(file) - 1] = '\0';
|
||||
}
|
||||
hts_choplastchar(file);
|
||||
if (strnotempty(file))
|
||||
file[strlen(file) - 1] = '\0';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3774,7 +3828,8 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
|
||||
strcpybuff(codebase, heap(ptr)->fil);
|
||||
else
|
||||
strcpybuff(codebase, heap(heap(ptr)->precedent)->fil);
|
||||
a = hts_lastcharptr(codebase);
|
||||
// empty codebase has no last char; codebase-1 would underflow
|
||||
a = codebase[0] != '\0' ? codebase + strlen(codebase) - 1 : codebase;
|
||||
while((*a) && (*a != '/') && (a > codebase))
|
||||
a--;
|
||||
if (*a == '/')
|
||||
|
||||
@@ -376,8 +376,8 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
|
||||
const char *template_footer, const char *adr,
|
||||
const char *fil);
|
||||
|
||||
// Flush ht_buff[0..ht_len] to save on disk; *fp closed+NULLed on write.
|
||||
// Precondition: ht_len>0.
|
||||
// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
|
||||
// closed+NULLed on write. Precondition: ht_len>0.
|
||||
void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
|
||||
FILE **fp, const char *ht_buff, size_t ht_len,
|
||||
const char *adr, const char *fil, const char *save);
|
||||
|
||||
@@ -357,7 +357,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char BIGSTK tempo[HTS_CDLMAXSIZE];
|
||||
|
||||
strcpybuff(tempo, argv[na] + 1);
|
||||
if (hts_lastchar(tempo) != '"') {
|
||||
if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
|
||||
char BIGSTK s[HTS_CDLMAXSIZE];
|
||||
|
||||
sprintf(s, "Missing quote in %s", argv[na]);
|
||||
@@ -365,7 +365,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
hts_choplastchar(tempo);
|
||||
tempo[strlen(tempo) - 1] = '\0';
|
||||
/* tempo is argv[na] minus its surrounding quotes, so it fits in place
|
||||
*/
|
||||
strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
|
||||
@@ -863,7 +863,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char BIGSTK tempo[HTS_CDLMAXSIZE + 256];
|
||||
|
||||
strcpybuff(tempo, argv[na] + 1);
|
||||
if (hts_lastchar(tempo) != '"') {
|
||||
if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
|
||||
char s[HTS_CDLMAXSIZE + 256];
|
||||
|
||||
sprintf(s, "Missing quote in %s", argv[na]);
|
||||
@@ -871,7 +871,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
hts_choplastchar(tempo);
|
||||
tempo[strlen(tempo) - 1] = '\0';
|
||||
/* tempo is argv[na] minus its surrounding quotes, so it fits in place
|
||||
*/
|
||||
strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
|
||||
@@ -2724,7 +2724,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
char *a;
|
||||
|
||||
strcpybuff(rpath, StringBuff(opt->path_html));
|
||||
hts_striplastchar(rpath, '/');
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/')
|
||||
rpath[strlen(rpath) - 1] = '\0';
|
||||
}
|
||||
a = strrchr(rpath, '/');
|
||||
if (a) {
|
||||
*a = '\0';
|
||||
|
||||
@@ -624,9 +624,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
} else {
|
||||
file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1,
|
||||
0);
|
||||
/* Every failure exit below would else leave the mirror truncated
|
||||
(#771); the resume branch appends and needs no backup. */
|
||||
back_refetch_backup(opt, back);
|
||||
back->r.fp = filecreate(&opt->state.strc, back->url_sav);
|
||||
}
|
||||
strcpybuff(back->info, "receiving");
|
||||
|
||||
@@ -83,8 +83,9 @@ void infomsg(const char *msg) {
|
||||
/* try the flag as-is, then strip a trailing N as the numeric-arg
|
||||
placeholder (cN -> c); this order keeps -%N from becoming -% */
|
||||
p = optreal_find(cmd);
|
||||
if (p < 0 && (int) strlen(cmd) > 2 && hts_lastchar(cmd) == 'N') {
|
||||
hts_striplastchar(cmd, 'N');
|
||||
if (p < 0 && (int) strlen(cmd) > 2 &&
|
||||
cmd[strlen(cmd) - 1] == 'N') {
|
||||
cmd[strlen(cmd) - 1] = '\0';
|
||||
p = optreal_find(cmd);
|
||||
}
|
||||
if (p >= 0) {
|
||||
@@ -210,7 +211,7 @@ void help_wizard(httrackp * opt) {
|
||||
strcatbuff(str, "/websites/");
|
||||
}
|
||||
if (strnotempty(str))
|
||||
if ((hts_lastchar(str) != '/') && (hts_lastchar(str) != '\\'))
|
||||
if ((str[strlen(str) - 1] != '/') && (str[strlen(str) - 1] != '\\'))
|
||||
strcatbuff(str, "/");
|
||||
strcatbuff(stropt2, "-O \"");
|
||||
strcatbuff(stropt2, str);
|
||||
|
||||
35
src/htslib.c
35
src/htslib.c
@@ -1322,7 +1322,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
p++; // sauter espaces
|
||||
if ((int) strlen(rcvd + p) < 250) { // pas trop long?
|
||||
char tmp[256];
|
||||
char *a = NULL;
|
||||
char *a = NULL, *b = NULL;
|
||||
|
||||
strcpybuff(tmp, rcvd + p);
|
||||
a = strstr(tmp, "filename=");
|
||||
@@ -1335,9 +1335,15 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
|
||||
while((c = strchr(a, '/'))) /* skip all / (see RFC2616) */
|
||||
a = c + 1;
|
||||
hts_rtrim(a, HTS_SPACES);
|
||||
if ((int) strlen(a) < 200) { // pas trop long?
|
||||
strcpybuff(retour->cdispo, a);
|
||||
b = a + strlen(a) - 1;
|
||||
while(is_space(*b))
|
||||
b--;
|
||||
b++;
|
||||
if (b) {
|
||||
*b = '\0';
|
||||
if ((int) strlen(a) < 200) { // pas trop long?
|
||||
strcpybuff(retour->cdispo, a);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3309,9 +3315,8 @@ int ishtml(httrackp * opt, const char *fil) {
|
||||
}
|
||||
|
||||
/* Search for known ext */
|
||||
for (a = hts_lastcharptr(fil_noquery);
|
||||
*a != '.' && *a != '/' && a > fil_noquery; a--)
|
||||
;
|
||||
for(a = fil_noquery + strlen(fil_noquery) - 1;
|
||||
*a != '.' && *a != '/' && a > fil_noquery; a--) ;
|
||||
if (*a == '.') { // a une extension
|
||||
char BIGSTK fil_noquery[HTS_URLMAXSIZE * 2];
|
||||
char *b;
|
||||
@@ -4248,8 +4253,9 @@ HTSEXT_API hts_boolean get_httptype_sized(httrackp *opt, char *s, size_t ssize,
|
||||
return 1;
|
||||
} else {
|
||||
/* Check html -> text/html */
|
||||
const char *a = hts_lastcharptr(fil);
|
||||
const char *a = fil + strlen(fil) - 1;
|
||||
|
||||
/* a < fil when fil is empty: bound before dereferencing */
|
||||
while ((a > fil) && (*a != '.') && (*a != '/'))
|
||||
a--;
|
||||
if (a >= fil && *a == '.' && strlen(a) < 32) {
|
||||
@@ -5072,7 +5078,7 @@ static int hts_dns_resolve_nocache_list(const char *const hostname,
|
||||
if (!strnotempty(hostname) || max <= 0) {
|
||||
return 0;
|
||||
}
|
||||
if ((hostname[0] == '[') && (hts_lastchar(hostname) == ']')) {
|
||||
if ((hostname[0] == '[') && (hostname[strlen(hostname) - 1] == ']')) {
|
||||
size_t size = strlen(hostname);
|
||||
char *copy = malloct(size + 1);
|
||||
int count;
|
||||
@@ -5485,8 +5491,9 @@ void cut_path(char *fullpath, char *path, size_t path_size, char *pname,
|
||||
size_t pname_size) {
|
||||
path[0] = pname[0] = '\0';
|
||||
if (strnotempty(fullpath)) {
|
||||
if (!hts_striplastchar(fullpath, '/'))
|
||||
hts_striplastchar(fullpath, '\\');
|
||||
if ((fullpath[strlen(fullpath) - 1] == '/')
|
||||
|| (fullpath[strlen(fullpath) - 1] == '\\'))
|
||||
fullpath[strlen(fullpath) - 1] = '\0';
|
||||
if (strlen(fullpath) > 1) {
|
||||
char *a;
|
||||
|
||||
@@ -5784,8 +5791,7 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_
|
||||
if (hts_log_print_callback != NULL) {
|
||||
va_list args_copy;
|
||||
va_copy(args_copy, args);
|
||||
/* the copy, so the vfprintf() below still has an unread list */
|
||||
hts_log_print_callback(opt, type, format, args_copy);
|
||||
hts_log_print_callback(opt, type, format, args);
|
||||
va_end(args_copy);
|
||||
}
|
||||
if (opt != NULL && opt->log != NULL) {
|
||||
@@ -6642,12 +6648,9 @@ int hts_rename_utf8(const char *oldpath, const char *newpath) {
|
||||
LPWSTR wnewpath = hts_pathToUCS2(newpath);
|
||||
if (woldpath != NULL && wnewpath != NULL) {
|
||||
const int result = _wrename(woldpath, wnewpath);
|
||||
/* Save errno: callers key off it (#779) and free() may clobber it. */
|
||||
const int err = errno;
|
||||
|
||||
free(woldpath);
|
||||
free(wnewpath);
|
||||
errno = err;
|
||||
return result;
|
||||
} else {
|
||||
if (woldpath != NULL)
|
||||
|
||||
@@ -437,7 +437,9 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
strcpybuff(fil_complete_patche, normfil);
|
||||
// Version avec ou sans /
|
||||
if (!hts_striplastchar(fil_complete_patche, '/'))
|
||||
if (fil_complete_patche[strlen(fil_complete_patche) - 1] == '/')
|
||||
fil_complete_patche[strlen(fil_complete_patche) - 1] = '\0';
|
||||
else
|
||||
strcatbuff(fil_complete_patche, "/");
|
||||
i = hash_read(hash, normadr, fil_complete_patche, HASH_STRUCT_ORIGINAL_ADR_PATH); // recherche table 2 (former->adr+former->fil)
|
||||
if (i >= 0) {
|
||||
@@ -513,8 +515,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
&& protocol != PROTOCOL_FTP
|
||||
) {
|
||||
// tester type avec requète HEAD si on ne connait pas le type du fichier
|
||||
if (!((opt->check_type == 1) &&
|
||||
(hts_lastchar(fil) == '/'))) // slash doit être html?
|
||||
if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/'))) // slash doit être html?
|
||||
if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
|
||||
ishtml(opt, fil) < 0) { // unsure whether it's html or a file
|
||||
// lire dans le cache
|
||||
@@ -809,7 +810,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// - - - DEBUT NOMMAGE - - -
|
||||
|
||||
// Donner nom par défaut?
|
||||
if (hts_lastchar(fil) == '/') {
|
||||
if (fil[strlen(fil) - 1] == '/') {
|
||||
if (!strfield(adr_complete, "ftp://")
|
||||
) {
|
||||
strcatbuff(fil, DEFAULT_HTML); // nommer page par défaut!!
|
||||
@@ -823,7 +824,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// Change the extension? e.g. php3 saved as html, cgi as html or gif/xbm
|
||||
// depending on the resolved type.
|
||||
if (ext_chg && !opt->no_type_change) {
|
||||
char *a = hts_lastcharptr(fil);
|
||||
char *a = fil + strlen(fil) - 1;
|
||||
|
||||
if ((opt->debug > 1) && (opt->log != NULL)) {
|
||||
if (ext_chg == 1)
|
||||
@@ -858,7 +859,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
}
|
||||
// Rechercher premier / et dernier .
|
||||
{
|
||||
const char *a = hts_lastcharptr(fil);
|
||||
const char *a = fil + strlen(fil) - 1;
|
||||
|
||||
// passer structures
|
||||
start_pos = fil;
|
||||
@@ -1203,29 +1204,29 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
switch (opt->savename_type % 100) {
|
||||
case 4:
|
||||
case 5:{ // séparer par types
|
||||
const char *a = hts_lastcharptr(fil);
|
||||
const char *a = fil + strlen(fil) - 1;
|
||||
|
||||
// passer structures
|
||||
while ((a > fil) && (*a != '/') && (*a != '\\'))
|
||||
a--;
|
||||
if ((*a == '/') || (*a == '\\'))
|
||||
a++;
|
||||
|
||||
// html?
|
||||
if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) {
|
||||
if (opt->savename_type % 100 == 5)
|
||||
strcatbuff(afs->save, "html/");
|
||||
} else {
|
||||
const char *a = hts_lastcharptr(fil);
|
||||
|
||||
while ((a > fil) && (*a != '/') && (*a != '.'))
|
||||
// passer structures
|
||||
while((a > fil) && (*a != '/') && (*a != '\\'))
|
||||
a--;
|
||||
if (*a != '.')
|
||||
strcatbuff(afs->save, "other");
|
||||
else
|
||||
strcatbuff(afs->save, a + 1);
|
||||
strcatbuff(afs->save, "/");
|
||||
}
|
||||
if ((*a == '/') || (*a == '\\'))
|
||||
a++;
|
||||
|
||||
// html?
|
||||
if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) {
|
||||
if (opt->savename_type % 100 == 5)
|
||||
strcatbuff(afs->save, "html/");
|
||||
} else {
|
||||
const char *a = fil + strlen(fil) - 1;
|
||||
|
||||
while((a > fil) && (*a != '/') && (*a != '.'))
|
||||
a--;
|
||||
if (*a != '.')
|
||||
strcatbuff(afs->save, "other");
|
||||
else
|
||||
strcatbuff(afs->save, a + 1);
|
||||
strcatbuff(afs->save, "/");
|
||||
}
|
||||
/*strcatbuff(save,a); */
|
||||
/* add name */
|
||||
ADD_STANDARD_NAME(0);
|
||||
@@ -1258,7 +1259,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
}
|
||||
afs->save[i + j] = '\0';
|
||||
// ajouter extension
|
||||
a = hts_lastcharptr(fil);
|
||||
a = fil + strlen(fil) - 1;
|
||||
while((a > fil) && (*a != '/') && (*a != '.'))
|
||||
a--;
|
||||
if (*a == '.') {
|
||||
@@ -1283,7 +1284,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
hts_lowcase(afs->save);
|
||||
|
||||
if (hts_lastchar(afs->save) == '/')
|
||||
if (afs->save[strlen(afs->save) - 1] == '/')
|
||||
strcatbuff(afs->save, DEFAULT_HTML); // nommer page par défaut!!
|
||||
}
|
||||
|
||||
@@ -1303,7 +1304,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// cela évite les /chez/toto et les /chez/toto/index.html incompatibles
|
||||
if (opt->savename_type != -1 &&
|
||||
opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD) {
|
||||
char *a = hts_lastcharptr(afs->save);
|
||||
char *a = afs->save + strlen(afs->save) - 1;
|
||||
|
||||
while((a > afs->save) && (*a != '.') && (*a != '/'))
|
||||
a--;
|
||||
@@ -1420,10 +1421,8 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (opt->savename_83 > 0) {
|
||||
char *a, *last;
|
||||
|
||||
for (last = hts_lastcharptr(afs->save);
|
||||
last != afs->save && *last != '/' && *last != '\\' && *last != '.';
|
||||
last--)
|
||||
;
|
||||
for(last = afs->save + strlen(afs->save) - 1;
|
||||
last != afs->save && *last != '/' && *last != '\\' && *last != '.'; last--) ;
|
||||
if (*last != '.') {
|
||||
last = NULL;
|
||||
}
|
||||
@@ -1678,8 +1677,8 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
#endif
|
||||
} else { // utilisé par un AUTRE, changer de nom
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
char *a = hts_lastcharptr(afs->save);
|
||||
size_t stem;
|
||||
char *a = afs->save + strlen(afs->save) - 1;
|
||||
char *b;
|
||||
int n = 2;
|
||||
char collisionSeparator =
|
||||
((opt->savename_83 != HTS_SAVENAME_83_ISO9660) ? '-' : '_');
|
||||
@@ -1701,16 +1700,18 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
strcatbuff(tempo, afs->save);
|
||||
|
||||
// tester la présence d'un -xx (ex: index-2.html -> index-3.html)
|
||||
stem = hts_rtrimlen(tempo, "0123456789");
|
||||
if (stem != 0 && tempo[stem - 1] == collisionSeparator) {
|
||||
sscanf(tempo + stem, "%d", &n);
|
||||
tempo[stem - 1] = '\0'; // couper
|
||||
b = tempo + strlen(tempo) - 1;
|
||||
while(isdigit((unsigned char) *b))
|
||||
b--;
|
||||
if (*b == collisionSeparator) {
|
||||
sscanf(b + 1, "%d", &n);
|
||||
*b = '\0'; // couper
|
||||
n++; // plus un
|
||||
}
|
||||
// en plus il faut gérer le 8-3 .. pas facile le client
|
||||
if (opt->savename_83) {
|
||||
int max;
|
||||
char *a = hts_lastcharptr(tempo);
|
||||
char *a = tempo + strlen(tempo) - 1;
|
||||
|
||||
while((a > tempo) && (*a != '/'))
|
||||
a--;
|
||||
|
||||
@@ -1701,8 +1701,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
#endif
|
||||
) // ok pas de problème
|
||||
url_ok = 1;
|
||||
else if (hts_lastchar(tempo) ==
|
||||
'/') { // un slash: ok..
|
||||
else if (tempo[strlen(tempo) - 1] == '/') { // un slash: ok..
|
||||
if (inscript) // sinon si pas javascript, méfiance (répertoire style base?)
|
||||
url_ok = 1;
|
||||
}
|
||||
@@ -2006,8 +2005,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
if (eadr - html - 1 < HTS_URLMAXSIZE) { // pas trop long?
|
||||
strncpy(lien, html, eadr - html - 1);
|
||||
lien[eadr - html - 1] = '\0';
|
||||
while (hts_striplastchar(lien, ' ')) {
|
||||
}
|
||||
// supprimer les espaces
|
||||
while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
|
||||
lien[strlen(lien) - 1] = '\0';
|
||||
|
||||
} else
|
||||
lien[0] = '\0'; // erreur
|
||||
@@ -2057,7 +2057,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
q = strchr(a, '?'); // ne pas traiter après '?'
|
||||
if (!q)
|
||||
q = hts_lastcharptr(a);
|
||||
q = a + strlen(a) - 1;
|
||||
while((p = strstr(a, "//")) && (!done)) { // remplacer // par /
|
||||
if (p > q) { // après le ? (toto.cgi?param=1//2.3)
|
||||
done = 1; // stopper
|
||||
@@ -2207,8 +2207,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// supposition dangereuse?
|
||||
// OUI!!
|
||||
#if HTS_TILDE_SLASH
|
||||
if (hts_lastchar(lien) != '/') {
|
||||
char *a = hts_lastcharptr(lien);
|
||||
if (lien[strlen(lien) - 1] != '/') {
|
||||
char *a = lien + strlen(lien) - 1;
|
||||
|
||||
// éviter aussi index~1.html
|
||||
while(a > lien && (*a != '~') && (*a != '/')
|
||||
@@ -2254,7 +2254,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// vérifier que l'on ne doit pas ajouter de .class
|
||||
if (!error) {
|
||||
if (add_class) {
|
||||
char *a = hts_lastcharptr(lien);
|
||||
char *a = lien + strlen(lien) - 1;
|
||||
|
||||
while((a > lien) && (*a != '/') && (*a != '.'))
|
||||
a--;
|
||||
@@ -2272,7 +2272,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
// Vérifier les codebase=applet (au lieu de applet/)
|
||||
if (p_type == -2) { // codebase
|
||||
if (strnotempty(lien)) {
|
||||
if (hts_lastchar(lien) != '/') { // pas répertoire
|
||||
if (lien[strlen(lien) - 1] != '/') { // pas répertoire
|
||||
strcatbuff(lien, "/");
|
||||
}
|
||||
}
|
||||
@@ -2309,7 +2309,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
{
|
||||
char *a;
|
||||
|
||||
a = hts_lastcharptr(lien);
|
||||
a = lien + strlen(lien) - 1;
|
||||
while((*a) && (*a != '/') && (a > lien))
|
||||
a--;
|
||||
if (*a == '/') {
|
||||
@@ -2320,8 +2320,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
}
|
||||
|
||||
if (!error) { // pas d'erreur?
|
||||
if (p_type == 2) { // code ET PAS codebase
|
||||
char *a = hts_lastcharptr(lien);
|
||||
if (p_type == 2) { // code ET PAS codebase
|
||||
char *a = lien + strlen(lien) - 1;
|
||||
char *start_of_filename = jump_identification(lien);
|
||||
|
||||
if (start_of_filename != NULL
|
||||
@@ -2688,11 +2688,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
int cat_data_len = 0;
|
||||
|
||||
// ajouter lien external
|
||||
switch ((link_has_authority(afs.af.adr))
|
||||
? 1
|
||||
: ((hts_lastchar(afs.af.fil) == '/')
|
||||
? 1
|
||||
: (ishtml(opt, afs.af.fil)))) {
|
||||
switch ((link_has_authority(afs.af.adr)) ? 1
|
||||
: ((afs.af.fil[strlen(afs.af.fil) - 1] ==
|
||||
'/') ? 1 : (ishtml(opt, afs.af.fil)))) {
|
||||
case 1:
|
||||
case -2: // html ou répertoire
|
||||
if (opt->getmode & HTS_GETMODE_HTML) {
|
||||
@@ -2735,7 +2733,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
cat_data_len = HTS_DATA_UNKNOWN_HTML_LEN;
|
||||
}
|
||||
break;
|
||||
} // html,gif
|
||||
} // html,gif
|
||||
|
||||
if (patch_it) {
|
||||
char BIGSTK save[HTS_URLMAXSIZE * 2];
|
||||
|
||||
@@ -539,68 +539,6 @@ static HTS_INLINE HTS_UNUSED HTS_PRINTF_FUN(3, 4) void slprintfbuff_clip(
|
||||
#define sprintfbuff(ARR, ...) slprintfbuff((ARR), sizeof(ARR), __VA_ARGS__)
|
||||
#endif
|
||||
|
||||
/* Last character of s, or '\0' when s is empty. Replaces s[strlen(s) - 1],
|
||||
which indexes one byte before the buffer on an empty string. */
|
||||
static HTS_INLINE HTS_UNUSED char hts_lastchar(const char *s) {
|
||||
const size_t len = strlen(s);
|
||||
|
||||
return len != 0 ? s[len - 1] : '\0';
|
||||
}
|
||||
|
||||
/* Drop a trailing c from s if present; HTS_TRUE if one was dropped. */
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean hts_striplastchar(char *s, char c) {
|
||||
const size_t len = strlen(s);
|
||||
|
||||
if (len != 0 && s[len - 1] == c) {
|
||||
s[len - 1] = '\0';
|
||||
return HTS_TRUE;
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Drop the last character of s whatever it is; HTS_TRUE if s was not empty. */
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean hts_choplastchar(char *s) {
|
||||
const size_t len = strlen(s);
|
||||
|
||||
if (len != 0) {
|
||||
s[len - 1] = '\0';
|
||||
return HTS_TRUE;
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
/* htslib.h's is_space() and is_realspace() as hts_rtrim() sets; -#test=rtrim
|
||||
keeps them in sync. */
|
||||
#define HTS_SPACES " \"\n\r\t\f\v'"
|
||||
#define HTS_REALSPACES " \n\r\t\f\v"
|
||||
|
||||
/* Length of s once its trailing bytes from set are dropped; 0 if they all are.
|
||||
Counts down from the end, so it stops at s rather than below the buffer. */
|
||||
static HTS_INLINE HTS_UNUSED size_t hts_rtrimlen(const char *s,
|
||||
const char *set) {
|
||||
size_t len = strlen(s);
|
||||
|
||||
while (len != 0 && strchr(set, s[len - 1]) != NULL)
|
||||
len--;
|
||||
return len;
|
||||
}
|
||||
|
||||
/* Drop the trailing bytes of s that occur in set. */
|
||||
static HTS_INLINE HTS_UNUSED void hts_rtrim(char *s, const char *set) {
|
||||
s[hts_rtrimlen(s, set)] = '\0';
|
||||
}
|
||||
|
||||
/* Offset of the last character of s, or 0 when s is empty. */
|
||||
static HTS_INLINE HTS_UNUSED size_t hts_lastcharoffset(const char *s) {
|
||||
const size_t len = strlen(s);
|
||||
|
||||
return len != 0 ? len - 1 : 0;
|
||||
}
|
||||
|
||||
/* Address of the last character of S, or of its terminating NUL when S is
|
||||
empty. S is evaluated twice, so pass an lvalue. */
|
||||
#define hts_lastcharptr(S) ((S) + hts_lastcharoffset(S))
|
||||
|
||||
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
|
||||
added bounds checking. freet() also NULLs the freed pointer and tolerates
|
||||
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the
|
||||
|
||||
@@ -2622,71 +2622,6 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static char st_log_callback_seen[256];
|
||||
|
||||
static void st_log_callback(httrackp *opt, int type, const char *format,
|
||||
va_list args) {
|
||||
(void) opt;
|
||||
(void) type;
|
||||
(void) vsnprintf(st_log_callback_seen, sizeof(st_log_callback_seen), format,
|
||||
args);
|
||||
}
|
||||
|
||||
/* The callback must not consume the va_list the log file's vfprintf() needs. */
|
||||
static int st_logcallback(httrackp *opt, int argc, char **argv) {
|
||||
static const char want[] = "42 sentinel";
|
||||
static const char want_filtered[] = "7 filtered";
|
||||
char BIGSTK seen[sizeof(st_log_callback_seen)];
|
||||
char BIGSTK line[256];
|
||||
FILE *fp;
|
||||
int rc = 1;
|
||||
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
fp = tmpfile();
|
||||
if (fp == NULL) {
|
||||
fprintf(stderr, "logcallback: tmpfile() failed\n");
|
||||
return 1;
|
||||
}
|
||||
opt->log = fp;
|
||||
opt->debug = LOG_NOTICE;
|
||||
st_log_callback_seen[0] = '\0';
|
||||
hts_set_log_vprint_callback(st_log_callback);
|
||||
hts_log_print(opt, LOG_NOTICE, "%d %s", 42, "sentinel");
|
||||
hts_set_log_vprint_callback(NULL);
|
||||
opt->log = NULL;
|
||||
strcpybuff(seen, st_log_callback_seen);
|
||||
|
||||
rewind(fp);
|
||||
line[0] = '\0';
|
||||
(void) fgets(line, (int) sizeof(line), fp);
|
||||
fclose(fp);
|
||||
|
||||
/* The callback runs above the level filter and without a log file at all;
|
||||
the front-ends that install one usually have no opt->log open. */
|
||||
st_log_callback_seen[0] = '\0';
|
||||
hts_set_log_vprint_callback(st_log_callback);
|
||||
hts_log_print(opt, LOG_DEBUG, "%d %s", 7, "filtered");
|
||||
hts_set_log_vprint_callback(NULL);
|
||||
|
||||
/* Same arguments both ways; the file line carries a level prefix. */
|
||||
if (strcmp(seen, want) != 0)
|
||||
fprintf(stderr, "logcallback: callback got '%s' want '%s'\n", seen, want);
|
||||
else if (strstr(line, want) == NULL)
|
||||
fprintf(stderr, "logcallback: log file got '%s' want it to carry '%s'\n",
|
||||
line, want);
|
||||
else if (strcmp(st_log_callback_seen, want_filtered) != 0)
|
||||
fprintf(stderr, "logcallback: unfiltered callback got '%s' want '%s'\n",
|
||||
st_log_callback_seen, want_filtered);
|
||||
else
|
||||
rc = 0;
|
||||
|
||||
if (rc == 0)
|
||||
printf("logcallback self-test OK\n");
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* an empty fil started htsAddLink's codebase walk before the buffer (#730) */
|
||||
static int st_addlink(httrackp *opt, int argc, char **argv) {
|
||||
htsmoduleStruct BIGSTK str;
|
||||
@@ -5009,101 +4944,6 @@ static int st_warc_surt(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/* A URL longer than the old 1024-byte header-format buffer must still reach the
|
||||
archive: the record used to be abandoned whole, silently (#785). The sweep
|
||||
straddles the boundary so both the stack-buffer and the grow path run. */
|
||||
static int st_warc_longurl(httrackp *opt, int argc, char **argv) {
|
||||
/* "WARC-Target-URI: " + CRLF costs 19 bytes, so the old buffer failed at
|
||||
1005; 9000 forces several reallocs within one record. */
|
||||
static const size_t lengths[] = {100, 1003, 1004, 1005, 1006, 2000, 9000};
|
||||
static const char resp_hdr[] =
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n";
|
||||
char path[HTS_URLMAXSIZE * 2];
|
||||
char body[64];
|
||||
warc_writer *w;
|
||||
FILE *fp;
|
||||
char *blob;
|
||||
LLint fsz;
|
||||
const char *at2;
|
||||
size_t i, n, nrec = 0;
|
||||
int err = 0;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-longurl: need a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
snprintf(path, sizeof(path), "%s/longurl.warc", argv[0]);
|
||||
|
||||
w = warc_open(opt, path);
|
||||
if (w == NULL) {
|
||||
fprintf(stderr, "warc-longurl: could not create %s\n", path);
|
||||
return 1;
|
||||
}
|
||||
for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
|
||||
const size_t len = lengths[i];
|
||||
char *uri = malloct(len + 1);
|
||||
|
||||
if (uri == NULL) {
|
||||
warc_close(w);
|
||||
return 1;
|
||||
}
|
||||
/* A distinct tail per URI so a truncated one cannot match another. */
|
||||
snprintf(uri, len + 1, "http://example.com/%04d/", (int) len);
|
||||
memset(uri + strlen(uri), 'a', len - strlen(uri));
|
||||
uri[len] = '\0';
|
||||
/* Distinct payloads: identical ones dedupe into revisit records. */
|
||||
snprintf(body, sizeof(body), "<html><body>%04d</body></html>\n", (int) len);
|
||||
if (warc_write_transaction(w, uri, NULL, NULL, resp_hdr, body, strlen(body),
|
||||
NULL, 200, 0, 0) != 0) {
|
||||
fprintf(stderr, "warc-longurl: write failed at length %d\n", (int) len);
|
||||
err = 1;
|
||||
}
|
||||
freet(uri);
|
||||
}
|
||||
warc_close(w);
|
||||
|
||||
fsz = fsize_utf8(path);
|
||||
blob = (fsz > 0) ? malloct((size_t) fsz + 1) : NULL;
|
||||
if (blob == NULL) {
|
||||
fprintf(stderr, "warc-longurl: no archive written\n");
|
||||
return 1;
|
||||
}
|
||||
fp = FOPEN(path, "rb");
|
||||
n = (fp != NULL) ? fread(blob, 1, (size_t) fsz, fp) : 0;
|
||||
if (fp != NULL)
|
||||
fclose(fp);
|
||||
blob[n] = '\0';
|
||||
|
||||
for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
|
||||
const size_t len = lengths[i];
|
||||
char want[64];
|
||||
const char *at;
|
||||
|
||||
snprintf(want, sizeof(want), "WARC-Target-URI: http://example.com/%04d/",
|
||||
(int) len);
|
||||
at = strstr(blob, want);
|
||||
if (at == NULL) {
|
||||
fprintf(stderr, "warc-longurl: length %d lost its record\n", (int) len);
|
||||
err = 1;
|
||||
} else if (strlen(at) < strlen("WARC-Target-URI: ") + len ||
|
||||
at[strlen("WARC-Target-URI: ") + len] != '\r') {
|
||||
fprintf(stderr, "warc-longurl: length %d truncated\n", (int) len);
|
||||
err = 1;
|
||||
}
|
||||
}
|
||||
for (at2 = blob; (at2 = strstr(at2, "WARC-Type: response")) != NULL; at2++)
|
||||
nrec++;
|
||||
if (nrec != sizeof(lengths) / sizeof(lengths[0])) {
|
||||
fprintf(stderr, "warc-longurl: %d response records, want %d\n", (int) nrec,
|
||||
(int) (sizeof(lengths) / sizeof(lengths[0])));
|
||||
err = 1;
|
||||
}
|
||||
|
||||
freet(blob);
|
||||
printf("warc-longurl: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* End-to-end CDXJ: crawl a handful of records with --warc-cdx, then verify the
|
||||
.cdx is sorted, has exactly one line per response/revisit/resource (none for
|
||||
warcinfo/request), and each offset/length points at a gzip member that
|
||||
@@ -6115,104 +5955,6 @@ static int st_mirrorio(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void ro_put(const char *path, const char *data) {
|
||||
FILE *const fp = FOPEN(path, "wb");
|
||||
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, strlen(data), fp) == strlen(data));
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* HTS_TRUE if path holds exactly data. */
|
||||
static hts_boolean ro_is(const char *path, const char *data) {
|
||||
char buf[64];
|
||||
FILE *const fp = FOPEN(path, "rb");
|
||||
size_t n;
|
||||
|
||||
if (fp == NULL)
|
||||
return HTS_FALSE;
|
||||
n = fread(buf, 1, sizeof(buf), fp);
|
||||
fclose(fp);
|
||||
return n == strlen(data) && memcmp(buf, data, n) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
// -#test=renameover <dir>: hts_rename_over() must replace an existing dst and
|
||||
// never delete one it did not replace (#779). Which half is live depends on
|
||||
// what rename() does to an existing target, so probe that and name the regime.
|
||||
static int st_renameover(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "renameover: needs a writable base dir\n");
|
||||
return 1;
|
||||
}
|
||||
char src[HTS_URLMAXSIZE * 2], dst[HTS_URLMAXSIZE * 2];
|
||||
int err = 0;
|
||||
|
||||
fconcat(src, sizeof(src), argv[0], "renameover-src.bin");
|
||||
fconcat(dst, sizeof(dst), argv[0], "renameover-dst.bin");
|
||||
|
||||
(void) UNLINK(src);
|
||||
(void) UNLINK(dst);
|
||||
ro_put(src, "probe");
|
||||
ro_put(dst, "probe");
|
||||
|
||||
const int probe = RENAME(src, dst) == 0 ? 0 : errno;
|
||||
/* Only a target in the way is something the unlink can clear. */
|
||||
const hts_boolean replaceable = probe == 0 || probe == EEXIST;
|
||||
|
||||
printf("renameover: regime %s\n",
|
||||
probe == 0 ? "clobber" : (probe == EEXIST ? "fallback" : "refused"));
|
||||
|
||||
(void) UNLINK(src);
|
||||
(void) UNLINK(dst);
|
||||
ro_put(src, "new");
|
||||
ro_put(dst, "old");
|
||||
if (replaceable) {
|
||||
/* An existing dst must still be replaced: the unlink is for this. */
|
||||
if (!hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: replacing an existing dst failed: %s\n",
|
||||
strerror(errno));
|
||||
err++;
|
||||
} else if (!ro_is(dst, "new") || fexist_utf8(src)) {
|
||||
fprintf(stderr, "renameover: dst was not replaced by src\n");
|
||||
err++;
|
||||
}
|
||||
} else {
|
||||
/* A failure the unlink cannot fix must leave dst as it was. */
|
||||
if (hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: an unfixable failure reported success\n");
|
||||
err++;
|
||||
}
|
||||
if (!ro_is(dst, "old")) {
|
||||
fprintf(stderr, "renameover: an unfixable failure destroyed dst\n");
|
||||
err++;
|
||||
}
|
||||
}
|
||||
|
||||
/* A missing src must leave dst alone and report failure. */
|
||||
(void) UNLINK(src);
|
||||
ro_put(dst, "keep");
|
||||
if (hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: a missing src reported success\n");
|
||||
err++;
|
||||
}
|
||||
if (!ro_is(dst, "keep")) {
|
||||
fprintf(stderr, "renameover: a missing src destroyed dst\n");
|
||||
err++;
|
||||
}
|
||||
|
||||
/* Same, with dst absent too: nothing to lose, still a failure. */
|
||||
(void) UNLINK(dst);
|
||||
if (hts_rename_over(src, dst)) {
|
||||
fprintf(stderr, "renameover: a missing src and dst reported success\n");
|
||||
err++;
|
||||
}
|
||||
|
||||
(void) UNLINK(dst);
|
||||
printf("renameover: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
// -#test=direnum <dir>: enumerate a long+non-ASCII directory via the
|
||||
// opendir/readdir wrappers; children must round-trip as UTF-8 (#133,#630).
|
||||
static int st_direnum(httrackp *opt, int argc, char **argv) {
|
||||
@@ -6526,13 +6268,6 @@ static void threadwait_gated_thread(void *arg) {
|
||||
hts_mutexrelease(&threadwait_lock);
|
||||
}
|
||||
|
||||
static int st_backswap(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
return back_selftest_slot_swap();
|
||||
}
|
||||
|
||||
static int st_threadwait(httrackp *opt, int argc, char **argv) {
|
||||
int err = 0;
|
||||
int i, round;
|
||||
@@ -6639,7 +6374,7 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
|
||||
return 1;
|
||||
}
|
||||
strcpybuff(base, argv[0]);
|
||||
if (base[0] != '\0' && hts_lastchar(base) != '/')
|
||||
if (base[0] != '\0' && base[strlen(base) - 1] != '/')
|
||||
strcatbuff(base, "/");
|
||||
StringCopy(opt->path_html, base);
|
||||
StringCopy(opt->path_html_utf8, base);
|
||||
@@ -6710,215 +6445,6 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/* The x[strlen(x) - 1] class (#770) and its pointer spelling x + strlen(x) - 1
|
||||
(#781). The string starts mid-arena so the byte it must not touch is a real
|
||||
neighbour; poisoned with '#', not 0, or a stray NUL terminator would read as
|
||||
untouched. */
|
||||
static int st_lastchar(httrackp *opt, int argc, char **argv) {
|
||||
enum { off = 8 };
|
||||
|
||||
char arena[16];
|
||||
char *const s = &arena[off];
|
||||
const int guard = off - 1; /* what the old idiom clobbers */
|
||||
int err = 0;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
#define REPOISON(str) \
|
||||
do { \
|
||||
memset(arena, '#', sizeof(arena)); \
|
||||
strlcpybuff(s, (str), sizeof(arena) - off); \
|
||||
} while (0)
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
printf(" FAIL line %d: %s\n", __LINE__, #cond); \
|
||||
err = 1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* the empty string: every helper must report "nothing" and touch nothing */
|
||||
REPOISON("");
|
||||
CHECK(hts_lastchar(s) == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
REPOISON("");
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
|
||||
CHECK(arena[guard] == '#');
|
||||
CHECK(s[0] == '\0');
|
||||
REPOISON("");
|
||||
CHECK(hts_choplastchar(s) == HTS_FALSE);
|
||||
CHECK(arena[guard] == '#');
|
||||
CHECK(s[0] == '\0');
|
||||
|
||||
/* a '/' sitting where the underflow would land must not be mistaken for the
|
||||
string's own last byte -- this is the #768 shape */
|
||||
REPOISON("");
|
||||
arena[guard] = '/';
|
||||
CHECK(hts_lastchar(s) == '\0');
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
|
||||
CHECK(arena[guard] == '/');
|
||||
|
||||
/* non-empty: ordinary behaviour */
|
||||
REPOISON("ab/");
|
||||
CHECK(hts_lastchar(s) == '/');
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_TRUE);
|
||||
CHECK(strcmp(s, "ab") == 0);
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
|
||||
CHECK(strcmp(s, "ab") == 0);
|
||||
CHECK(hts_choplastchar(s) == HTS_TRUE);
|
||||
CHECK(strcmp(s, "a") == 0);
|
||||
CHECK(hts_choplastchar(s) == HTS_TRUE);
|
||||
CHECK(s[0] == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
/* one-character string: the boundary the guards get wrong */
|
||||
REPOISON("/");
|
||||
CHECK(hts_lastchar(s) == '/');
|
||||
CHECK(hts_striplastchar(s, '/') == HTS_TRUE);
|
||||
CHECK(s[0] == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
/* the pointer spelling (#781): on an empty string the address must be the
|
||||
terminating NUL, never the byte before it */
|
||||
REPOISON("");
|
||||
CHECK(hts_lastcharoffset(s) == 0);
|
||||
CHECK(hts_lastcharptr(s) == s);
|
||||
CHECK(*hts_lastcharptr(s) == '\0');
|
||||
*hts_lastcharptr(s) = 'Z'; /* a write through it must stay inside s */
|
||||
CHECK(arena[guard] == '#');
|
||||
CHECK(s[0] == 'Z');
|
||||
|
||||
/* the neighbour must not be mistaken for the string's own last byte */
|
||||
REPOISON("");
|
||||
arena[guard] = '/';
|
||||
CHECK(hts_lastcharptr(s) == s);
|
||||
CHECK(*hts_lastcharptr(s) != '/');
|
||||
CHECK(arena[guard] == '/');
|
||||
|
||||
/* the walk-back loops the sites use must stop at once on an empty string */
|
||||
REPOISON("");
|
||||
{
|
||||
const char *p = hts_lastcharptr(s);
|
||||
int steps = 0;
|
||||
|
||||
while (p > s && *p != '/')
|
||||
p--, steps++;
|
||||
CHECK(steps == 0);
|
||||
CHECK(p == s);
|
||||
}
|
||||
|
||||
REPOISON("ab/");
|
||||
CHECK(hts_lastcharoffset(s) == 2);
|
||||
CHECK(hts_lastcharptr(s) == s + 2);
|
||||
CHECK(*hts_lastcharptr(s) == '/');
|
||||
REPOISON("/");
|
||||
CHECK(hts_lastcharptr(s) == s);
|
||||
CHECK(*hts_lastcharptr(s) == '/');
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
/* control: the canary must be able to fail, or the checks above prove
|
||||
nothing. Clobber it exactly as the unguarded idiom would. */
|
||||
REPOISON("");
|
||||
s[-1] = '\0';
|
||||
CHECK(arena[guard] != '#');
|
||||
REPOISON("");
|
||||
*(s + strlen(s) - 1) = 'X';
|
||||
CHECK(arena[guard] != '#');
|
||||
|
||||
#undef REPOISON
|
||||
#undef CHECK
|
||||
|
||||
printf("lastchar self-test: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* hts_rtrim() and the sets it is called with. The string starts mid-arena, and
|
||||
the byte below it is poisoned with '#' rather than 0, or the stray NUL the
|
||||
old loop wrote there would read as untouched. */
|
||||
static int st_rtrim(httrackp *opt, int argc, char **argv) {
|
||||
enum { off = 8 };
|
||||
|
||||
char arena[24];
|
||||
char *const s = &arena[off];
|
||||
const int guard = off - 1;
|
||||
int err = 0;
|
||||
int c;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
#define REPOISON(str) \
|
||||
do { \
|
||||
memset(arena, '#', sizeof(arena)); \
|
||||
strlcpybuff(s, (str), sizeof(arena) - off); \
|
||||
} while (0)
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
printf(" FAIL line %d: %s\n", __LINE__, #cond); \
|
||||
err = 1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* nothing but spaces: the case that ran the old loop off the front */
|
||||
REPOISON(" ");
|
||||
hts_rtrim(s, HTS_REALSPACES);
|
||||
CHECK(s[0] == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
/* a space sitting below the string must not be eaten as if it were part of
|
||||
it, which is exactly what the old loop did */
|
||||
REPOISON(" ");
|
||||
arena[guard] = ' ';
|
||||
hts_rtrim(s, HTS_REALSPACES);
|
||||
CHECK(s[0] == '\0');
|
||||
CHECK(arena[guard] == ' ');
|
||||
|
||||
REPOISON("");
|
||||
hts_rtrim(s, HTS_REALSPACES);
|
||||
CHECK(s[0] == '\0');
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
REPOISON("a b \t\r\n");
|
||||
hts_rtrim(s, HTS_REALSPACES);
|
||||
CHECK(strcmp(s, "a b") == 0);
|
||||
CHECK(arena[guard] == '#');
|
||||
|
||||
REPOISON("ab");
|
||||
hts_rtrim(s, HTS_REALSPACES);
|
||||
CHECK(strcmp(s, "ab") == 0);
|
||||
|
||||
/* quotes count as space for is_space() but not for is_realspace() */
|
||||
REPOISON("v\" ");
|
||||
hts_rtrim(s, HTS_REALSPACES);
|
||||
CHECK(strcmp(s, "v\"") == 0);
|
||||
REPOISON("v\" ");
|
||||
hts_rtrim(s, HTS_SPACES);
|
||||
CHECK(strcmp(s, "v") == 0);
|
||||
|
||||
/* the sets must stay the macros they stand for */
|
||||
for (c = 1; c < 256; c++) {
|
||||
const char b = (char) c;
|
||||
|
||||
CHECK((strchr(HTS_SPACES, b) != NULL) == (is_space(b) != 0));
|
||||
CHECK((strchr(HTS_REALSPACES, b) != NULL) == (is_realspace(b) != 0));
|
||||
}
|
||||
|
||||
/* control: the canary must be able to fail */
|
||||
REPOISON(" ");
|
||||
s[-1] = '\0';
|
||||
CHECK(arena[guard] != '#');
|
||||
|
||||
#undef REPOISON
|
||||
#undef CHECK
|
||||
|
||||
printf("rtrim self-test: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* Registry: name -> handler, with a usage hint and a one-line description. */
|
||||
/* ------------------------------------------------------------ */
|
||||
@@ -6974,18 +6500,12 @@ static const struct selftest_entry {
|
||||
{"strsafe", "[overflow|overflow-buff|overflow-src [str]]",
|
||||
"bounded string-op self-test", st_strsafe},
|
||||
{"copyopt", "", "copy_htsopt option-copy self-test", st_copyopt},
|
||||
{"lastchar", "",
|
||||
"last-char helpers never index before the buffer (#770, #781)",
|
||||
st_lastchar},
|
||||
{"rtrim", "", "hts_rtrim never walks below the buffer", st_rtrim},
|
||||
{"changes", "", "--changes bucket accounting and JSON escaping (#714)",
|
||||
st_changes},
|
||||
{"changes-race", "<dir>", "--changes under a late transfer thread (#714)",
|
||||
st_changes_race},
|
||||
{"threadwait", "", "htsthread_wait() joins threads spawned just before it",
|
||||
st_threadwait},
|
||||
{"backswap", "", "which backlog slots may be swapped to the ready table",
|
||||
st_backswap},
|
||||
{"pause", "", "randomized inter-file pause target self-test", st_pause},
|
||||
{"relative", "<link> <curr-file>", "relative link between two paths",
|
||||
st_relative},
|
||||
@@ -7018,8 +6538,6 @@ static const struct selftest_entry {
|
||||
st_growsize},
|
||||
{"addlink", "", "htsAddLink codebase walk over an empty current path",
|
||||
st_addlink},
|
||||
{"logcallback", "", "log callback must not consume the log file's va_list",
|
||||
st_logcallback},
|
||||
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
|
||||
{"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
|
||||
st_cacheindex},
|
||||
@@ -7078,9 +6596,6 @@ static const struct selftest_entry {
|
||||
st_warc_verbatim},
|
||||
{"warc-surt", "", "SURT canonicalization of the CDXJ sort key",
|
||||
st_warc_surt},
|
||||
{"warc-longurl", "<dir>",
|
||||
"a URL past the header-format buffer still reaches the archive",
|
||||
st_warc_longurl},
|
||||
{"longpath", "<dir>",
|
||||
"round-trip a >MAX_PATH file through the _w* wrappers (\\\\?\\ on "
|
||||
"Windows)",
|
||||
@@ -7088,10 +6603,6 @@ static const struct selftest_entry {
|
||||
{"mirrorio", "<dir>",
|
||||
"round-trip a long+non-ASCII path through the mirror I/O wrappers",
|
||||
st_mirrorio},
|
||||
{"renameover", "<dir>",
|
||||
"hts_rename_over(): replace dst, but never delete a dst it did not "
|
||||
"replace",
|
||||
st_renameover},
|
||||
{"direnum", "<dir>",
|
||||
"enumerate a long+non-ASCII directory through opendir/readdir",
|
||||
st_direnum},
|
||||
|
||||
@@ -1050,9 +1050,9 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
if (!linput(fp, line, sizeof(line) - 2)) {
|
||||
*str = '\0';
|
||||
}
|
||||
if (hts_lastchar(str) == '\\') {
|
||||
if (*str && str[strlen(str) - 1] == '\\') {
|
||||
nocr = 1;
|
||||
hts_striplastchar(str, '\\');
|
||||
str[strlen(str) - 1] = '\0';
|
||||
}
|
||||
while(*str) {
|
||||
char *pos;
|
||||
@@ -1169,8 +1169,11 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
char *rpath = (char *) adr;
|
||||
|
||||
//find_handle h;
|
||||
/* note: patching stored (inhash) value */
|
||||
hts_striplastchar(rpath, '/');
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/') {
|
||||
rpath[strlen(rpath) - 1] = '\0'; /* note: patching stored (inhash) value */
|
||||
}
|
||||
}
|
||||
{
|
||||
const char *profiles = hts_getcategories(rpath, 0);
|
||||
const char *categ = hts_getcategories(rpath, 1);
|
||||
|
||||
@@ -324,7 +324,8 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
curr = _curr;
|
||||
strlcpybuff(curr, curr_fil, sizeof(_curr));
|
||||
if ((a = strchr(curr, '?')) == NULL) { // cut at the ? (query parameters)
|
||||
a = hts_lastcharptr(curr);
|
||||
// an empty path has no last character: curr-1 would read before the buffer
|
||||
a = curr[0] != '\0' ? curr + strlen(curr) - 1 : curr;
|
||||
}
|
||||
while((*a != '/') && (a > curr))
|
||||
a--; // chercher dernier / du chemin courant
|
||||
@@ -984,7 +985,10 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
|
||||
&& toptemplate_bodycat) {
|
||||
|
||||
strcpybuff(rpath, path);
|
||||
hts_striplastchar(rpath, '/');
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/')
|
||||
rpath[strlen(rpath) - 1] = '\0';
|
||||
}
|
||||
|
||||
fpo = fopen(fconcat(catbuff, sizeof(catbuff), rpath, "/index.html"), "wb");
|
||||
if (fpo) {
|
||||
@@ -1197,8 +1201,11 @@ HTSEXT_API char *hts_getcategories(char *path, int type) {
|
||||
find_handle h;
|
||||
coucal hashCateg = NULL;
|
||||
|
||||
/* note: patching stored (inhash) value */
|
||||
hts_striplastchar(rpath, '/');
|
||||
if (rpath[0]) {
|
||||
if (rpath[strlen(rpath) - 1] == '/') {
|
||||
rpath[strlen(rpath) - 1] = '\0'; /* note: patching stored (inhash) value */
|
||||
}
|
||||
}
|
||||
h = hts_findfirst(rpath);
|
||||
if (h) {
|
||||
String iname = STRING_EMPTY;
|
||||
@@ -1295,7 +1302,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
|
||||
|
||||
strcpybuff(rpath, path);
|
||||
if (rpath[0]) {
|
||||
if (hts_lastchar(rpath) != '\\')
|
||||
if (rpath[strlen(rpath) - 1] != '\\')
|
||||
strcatbuff(rpath, "\\");
|
||||
}
|
||||
strcatbuff(rpath, "*.*");
|
||||
@@ -1307,7 +1314,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
|
||||
strcpybuff(find->path, path);
|
||||
{
|
||||
if (find->path[0]) {
|
||||
if (hts_lastchar(find->path) != '/')
|
||||
if (find->path[strlen(find->path) - 1] != '/')
|
||||
strcatbuff(find->path, "/");
|
||||
}
|
||||
}
|
||||
@@ -1444,14 +1451,7 @@ hts_boolean hts_rename_over(const char *src, const char *dst) {
|
||||
fconv(cdst, sizeof(cdst), dst);
|
||||
if (RENAME(csrc, cdst) == 0)
|
||||
return HTS_TRUE;
|
||||
/* Only a dst in the way is something the unlink can clear, and the CRT maps
|
||||
that to EEXIST; it keeps EACCES for a src another process holds, where
|
||||
removing dst would lose a file the retry cannot replace (#790). The src
|
||||
check covers a CRT that reports neither. */
|
||||
const int err = errno;
|
||||
|
||||
if (err != EEXIST || !fexist_utf8(src))
|
||||
return HTS_FALSE;
|
||||
/* RENAME does not clobber an existing target on Windows. */
|
||||
(void) UNLINK(cdst);
|
||||
return RENAME(csrc, cdst) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
@@ -138,9 +138,7 @@ HTSEXT_API hts_boolean hts_findisfile(find_handle find);
|
||||
HTSEXT_API hts_boolean hts_findissystem(find_handle find);
|
||||
|
||||
/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both
|
||||
paths are fconv()'d. dst is removed only to make room for a src that exists,
|
||||
so a caller whose src was never written keeps its dst; a retry that still
|
||||
fails does not (#790). */
|
||||
paths are fconv()'d. */
|
||||
hts_boolean hts_rename_over(const char *src, const char *dst);
|
||||
|
||||
#endif
|
||||
|
||||
199
src/htswarc.c
199
src/htswarc.c
@@ -57,9 +57,6 @@ Please visit our Website: http://www.httrack.com
|
||||
/* opt->state.warc value meaning "open failed once, do not retry". */
|
||||
#define WARC_DISABLED ((void *) ~(uintptr_t) 0)
|
||||
|
||||
/* Suffix of the in-progress archive when a previous one must survive it. */
|
||||
#define WARC_TMP_SUFFIX ".tmp"
|
||||
|
||||
struct warc_writer {
|
||||
FILE *f;
|
||||
httrackp *opt; /* kept for close-time logging (warc_wacz_package) */
|
||||
@@ -89,11 +86,6 @@ struct warc_writer {
|
||||
char *base_path; /* resolved archive path minus .warc[.gz] suffix */
|
||||
const char *base_ext; /* ".warc.gz" or ".warc" (static) */
|
||||
char *arc_path; /* full single-file archive path (NULL under rotation) */
|
||||
/* A re-run must not destroy an archive it cannot replace (#759). */
|
||||
hts_boolean protect_prev; /* previous archive present: build in a temp */
|
||||
hts_boolean opened; /* open completed; a failed one swaps nothing */
|
||||
hts_boolean failed; /* a record or segment was lost: swap nothing */
|
||||
uint64_t unbacked_revisits; /* revisits whose payload no file here holds */
|
||||
char **page_lines; /* one JSON page line per 200 text/html response, owned */
|
||||
size_t page_count;
|
||||
size_t page_cap;
|
||||
@@ -127,32 +119,24 @@ static void wbuf_free(wbuf *b) {
|
||||
b->len = b->cap = 0;
|
||||
}
|
||||
|
||||
/* Make room for n more bytes; returns 0 on success, -1 on OOM/overflow. */
|
||||
static int wbuf_reserve(wbuf *b, size_t n) {
|
||||
size_t ncap;
|
||||
char *nd;
|
||||
if (n > (size_t) -1 - b->len)
|
||||
return -1;
|
||||
if (b->len + n <= b->cap)
|
||||
return 0;
|
||||
ncap = b->cap ? b->cap : 256;
|
||||
while (ncap < b->len + n) {
|
||||
if (ncap > (size_t) -1 / 2)
|
||||
return -1;
|
||||
ncap *= 2;
|
||||
}
|
||||
nd = realloct(b->data, ncap);
|
||||
if (nd == NULL)
|
||||
return -1;
|
||||
b->data = nd;
|
||||
b->cap = ncap;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
|
||||
static int wbuf_add(wbuf *b, const void *p, size_t n) {
|
||||
if (wbuf_reserve(b, n) != 0)
|
||||
if (n > (size_t) -1 - b->len)
|
||||
return -1;
|
||||
if (b->len + n > b->cap) {
|
||||
size_t ncap = b->cap ? b->cap : 256;
|
||||
char *nd;
|
||||
while (ncap < b->len + n) {
|
||||
if (ncap > (size_t) -1 / 2)
|
||||
return -1;
|
||||
ncap *= 2;
|
||||
}
|
||||
nd = realloct(b->data, ncap);
|
||||
if (nd == NULL)
|
||||
return -1;
|
||||
b->data = nd;
|
||||
b->cap = ncap;
|
||||
}
|
||||
memcpy(b->data + b->len, p, n);
|
||||
b->len += n;
|
||||
return 0;
|
||||
@@ -164,32 +148,16 @@ static int wbuf_puts(wbuf *b, const char *s) {
|
||||
|
||||
static int wbuf_printf(wbuf *b, const char *fmt, ...) HTS_PRINTF_FUN(2, 3);
|
||||
|
||||
/* A header line carrying a URL has no useful bound, and giving up here loses
|
||||
the whole record, so anything past the stack buffer is formatted into the
|
||||
wbuf itself rather than rejected (#785). */
|
||||
static int wbuf_printf(wbuf *b, const char *fmt, ...) {
|
||||
char tmp[1024];
|
||||
size_t need;
|
||||
int n;
|
||||
va_list ap;
|
||||
va_start(ap, fmt);
|
||||
n = vsnprintf(tmp, sizeof(tmp), fmt, ap);
|
||||
va_end(ap);
|
||||
if (n < 0)
|
||||
if (n < 0 || (size_t) n >= sizeof(tmp))
|
||||
return -1;
|
||||
if ((size_t) n < sizeof(tmp))
|
||||
return wbuf_add(b, tmp, (size_t) n);
|
||||
need = (size_t) n + 1; /* +1: vsnprintf always writes the NUL */
|
||||
if (wbuf_reserve(b, need) != 0)
|
||||
return -1;
|
||||
va_start(ap, fmt);
|
||||
n = vsnprintf(b->data + b->len, need, fmt, ap);
|
||||
va_end(ap);
|
||||
/* Never advance past what was reserved, whatever the second pass returns. */
|
||||
if (n < 0 || (size_t) n >= need)
|
||||
return -1;
|
||||
b->len += (size_t) n; /* the NUL is scratch, overwritten by the next append */
|
||||
return 0;
|
||||
return wbuf_add(b, tmp, (size_t) n);
|
||||
}
|
||||
|
||||
/* ---- gzip-per-record member writer (mirrors ae_write_packed) ---- */
|
||||
@@ -1161,17 +1129,13 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
|
||||
never split a record, and never rotate a warcinfo (it opens a segment). */
|
||||
if (w->max_size > 0 && w->seg_base != NULL && w->offset >= w->max_size &&
|
||||
strcmp(type, "warcinfo") != 0) {
|
||||
if (warc_rotate(w) != 0) {
|
||||
w->failed = HTS_TRUE;
|
||||
if (warc_rotate(w) != 0)
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* F4: overflow-safe block length; http_hdr_len+sep is provably small. */
|
||||
if (payload > (size_t) -1 - http_hdr_len - sep) {
|
||||
w->failed = HTS_TRUE;
|
||||
if (payload > (size_t) -1 - http_hdr_len - sep)
|
||||
return -1;
|
||||
}
|
||||
block_len = http_hdr_len + sep + payload;
|
||||
|
||||
memset(&hdr, 0, sizeof(hdr));
|
||||
@@ -1277,23 +1241,11 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
|
||||
rc = 0;
|
||||
done:
|
||||
wbuf_free(&hdr);
|
||||
if (rc != 0)
|
||||
w->failed = HTS_TRUE; /* a truncated run must not replace a whole one */
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ---- segment rotation (--warc-max-size) ---- */
|
||||
|
||||
/* Path to open for the segment whose final path is `final`: that path itself,
|
||||
or a sibling temp while a previous archive must survive until close. */
|
||||
static const char *warc_open_path(warc_writer *w, const char *final, char *buf,
|
||||
size_t bufsz) {
|
||||
if (!w->protect_prev)
|
||||
return final;
|
||||
snprintf(buf, bufsz, "%s" WARC_TMP_SUFFIX, final);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Emit the warcinfo that heads a segment; sets w->info_id for its records. */
|
||||
static int warc_write_warcinfo_record(warc_writer *w) {
|
||||
w->info_id[0] = '\0'; /* warcinfo itself carries no WARC-Warcinfo-ID */
|
||||
@@ -1305,23 +1257,17 @@ static int warc_write_warcinfo_record(warc_writer *w) {
|
||||
|
||||
static int warc_rotate(warc_writer *w) {
|
||||
char namebuf[HTS_URLMAXSIZE * 2];
|
||||
char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const unsigned next = w->seg + 1;
|
||||
FILE *f;
|
||||
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, next,
|
||||
w->seg_ext);
|
||||
/* Open before advancing: w->seg must only ever name a segment that exists,
|
||||
or close-time packaging and swapping would work on a missing file. */
|
||||
f = FOPEN(fconv(catbuff, sizeof(catbuff),
|
||||
warc_open_path(w, namebuf, openbuf, sizeof(openbuf))),
|
||||
"wb");
|
||||
if (f == NULL)
|
||||
return -1;
|
||||
if (w->f != NULL)
|
||||
if (w->f != NULL) {
|
||||
fclose(w->f);
|
||||
w->f = f;
|
||||
w->seg = next;
|
||||
w->f = NULL;
|
||||
}
|
||||
w->seg++;
|
||||
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
|
||||
w->seg_ext);
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), namebuf), "wb");
|
||||
if (w->f == NULL)
|
||||
return -1;
|
||||
w->offset = 0;
|
||||
if (w->cdx_on) {
|
||||
freet(w->cur_seg);
|
||||
@@ -1378,7 +1324,6 @@ void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path) {
|
||||
warc_writer *warc_open(httrackp *opt, const char *path) {
|
||||
warc_writer *w;
|
||||
char namebuf[HTS_URLMAXSIZE * 2];
|
||||
char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
wbuf info;
|
||||
const char *robots;
|
||||
@@ -1529,105 +1474,35 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
|
||||
path = namebuf;
|
||||
}
|
||||
|
||||
if (w->max_size == 0 && (w->arc_path = strdupt(path)) == NULL) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
/* Set only once arc_path is recorded, so a half-built writer never swaps. */
|
||||
w->protect_prev = fsize_utf8(path) > 0 ? HTS_TRUE : HTS_FALSE;
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff),
|
||||
warc_open_path(w, path, openbuf, sizeof(openbuf))),
|
||||
"wb");
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
|
||||
if (w->f == NULL) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
if (w->cdx_on)
|
||||
w->cur_seg = path_basename_dup(path);
|
||||
if (w->wacz_on && w->max_size == 0)
|
||||
w->arc_path = strdupt(path); /* single-file: package this exact path */
|
||||
|
||||
if (warc_write_warcinfo_record(w) != 0) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
w->opened = HTS_TRUE;
|
||||
return w;
|
||||
}
|
||||
|
||||
/* Final path of segment s (the run's only archive when rotation is off). */
|
||||
static const char *warc_seg_path(warc_writer *w, unsigned s, char *buf,
|
||||
size_t bufsz) {
|
||||
if (w->max_size == 0)
|
||||
return w->arc_path;
|
||||
snprintf(buf, bufsz, "%s-%05u%s", w->seg_base, s, w->seg_ext);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Swap this run's archive into place, unless it only holds revisits naming
|
||||
bodies the previous archive still has and this one does not (#759).
|
||||
HTS_FALSE: the previous archive was kept, so leave its .cdx and .wacz too. */
|
||||
static hts_boolean warc_commit(warc_writer *w) {
|
||||
char finalbuf[HTS_URLMAXSIZE * 2];
|
||||
char tmpbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const unsigned nseg = (w->max_size > 0) ? w->seg + 1 : 1;
|
||||
hts_boolean swap;
|
||||
unsigned s;
|
||||
|
||||
if (!w->protect_prev)
|
||||
return HTS_TRUE; /* nothing was there to lose: written in place */
|
||||
|
||||
/* All or nothing: a swap stopping halfway would mix this run's segments with
|
||||
the previous one's, so require every temp before renaming any. */
|
||||
swap = w->opened && !w->failed && w->unbacked_revisits == 0;
|
||||
for (s = 0; s < nseg && swap; s++) {
|
||||
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
|
||||
warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
|
||||
swap = fsize_utf8(tmpbuf) > 0;
|
||||
}
|
||||
|
||||
if (swap) {
|
||||
for (s = 0; s < nseg; s++) {
|
||||
const char *final = warc_seg_path(w, s, finalbuf, sizeof(finalbuf));
|
||||
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX, final);
|
||||
if (!hts_rename_over(tmpbuf, final)) {
|
||||
hts_log_print(w->opt, LOG_ERROR | LOG_ERRNO,
|
||||
"WARC: could not replace %s", final);
|
||||
return HTS_FALSE;
|
||||
}
|
||||
}
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
for (s = 0; s < nseg; s++) {
|
||||
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
|
||||
warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
|
||||
(void) UNLINK(fconv(catbuff, sizeof(catbuff), tmpbuf));
|
||||
}
|
||||
if (w->unbacked_revisits > 0)
|
||||
hts_log_print(
|
||||
w->opt, LOG_ERROR,
|
||||
"WARC: this pass revisited %llu URL(s) without re-downloading them, so "
|
||||
"its archive would name bodies no file holds; kept the previous %s "
|
||||
"(re-run with -C0, or --warc-file with a name of its own)",
|
||||
(unsigned long long) w->unbacked_revisits,
|
||||
warc_seg_path(w, 0, finalbuf, sizeof(finalbuf)));
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
void warc_close(warc_writer *w) {
|
||||
size_t i;
|
||||
if (w == NULL)
|
||||
return;
|
||||
warc_cdx_flush(w); /* sort + write <base>.cdx before tearing down */
|
||||
if (w->f != NULL)
|
||||
fclose(w->f);
|
||||
w->f = NULL;
|
||||
if (warc_commit(w)) {
|
||||
warc_cdx_flush(w); /* sort + write <base>.cdx beside the archive */
|
||||
#if HTS_USEOPENSSL
|
||||
if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
|
||||
warc_wacz_package(w);
|
||||
if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
|
||||
warc_wacz_package(w);
|
||||
#endif
|
||||
}
|
||||
if (w->seen != NULL)
|
||||
coucal_delete(&w->seen);
|
||||
for (i = 0; i < w->cdx_count; i++)
|
||||
@@ -1708,12 +1583,6 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
|
||||
if (is_update_unchanged) {
|
||||
is_revisit = 1;
|
||||
profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
|
||||
/* Replay resolves a revisit by this field alone, and a 304 stands in for
|
||||
the same URL. No WARC-Refers-To-Date to go with it: the cache keeps the
|
||||
document's Last-Modified, never the previous capture time. */
|
||||
refers_uri = target_uri;
|
||||
/* Served from cache: the payload sits in the previous archive, not here. */
|
||||
w->unbacked_revisits++;
|
||||
} else if (have_pdig && w->seen != NULL) {
|
||||
void *prev = NULL;
|
||||
if (coucal_read_pvoid(w->seen, pdig, &prev) && prev != NULL) {
|
||||
|
||||
@@ -903,11 +903,10 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
|
||||
coucal hashtable = index->hash;
|
||||
|
||||
/* Compute base path for this index - the filename MUST be absolute! */
|
||||
for (slashes = 2, abpath = hts_lastcharptr(filename);
|
||||
abpath > filename &&
|
||||
((*abpath != '/' && *abpath != '\\') || --slashes > 0);
|
||||
abpath--)
|
||||
;
|
||||
for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
|
||||
abpath > filename && ((*abpath != '/' && *abpath != '\\')
|
||||
|| --slashes > 0);
|
||||
abpath--) ;
|
||||
index->path[0] = '\0';
|
||||
if (slashes == 0 && *abpath != 0) {
|
||||
int i;
|
||||
@@ -1500,11 +1499,10 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
|
||||
/* -------------------- COPY OF THE __New() CODE -------------------- */
|
||||
/* Compute base path for this index - the filename MUST be absolute! */
|
||||
for (slashes = 2, abpath = hts_lastcharptr(filename);
|
||||
abpath > filename &&
|
||||
((*abpath != '/' && *abpath != '\\') || --slashes > 0);
|
||||
abpath--)
|
||||
;
|
||||
for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
|
||||
abpath > filename && ((*abpath != '/' && *abpath != '\\')
|
||||
|| --slashes > 0);
|
||||
abpath--) ;
|
||||
index->path[0] = '\0';
|
||||
if (slashes == 0 && *abpath != 0) {
|
||||
int i;
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# A ready slot still owning a temporary must stay in memory: swapping it out
|
||||
# clears the entry, which unlinks the re-fetch backup (#771).
|
||||
httrack -O /dev/null -#test=backswap | grep -q "backswap self-test: OK"
|
||||
@@ -4,8 +4,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_changes_st.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmpdir"' EXIT
|
||||
trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# No pipe into grep: SIGPIPE would mask a failing exit status.
|
||||
expect_ok() {
|
||||
|
||||
@@ -11,8 +11,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_cmdline.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
echo '<html><body>hello</body></html>' >"$tmp/index.html"
|
||||
|
||||
|
||||
@@ -7,6 +7,6 @@ set -euo pipefail
|
||||
# cookies *@*.txt) from a long, non-ASCII folder through the UTF-8/long-path
|
||||
# file wrappers (#133,#630).
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=cookieimport "$dir" | grep -q "cookieimport:.*OK"
|
||||
|
||||
@@ -6,6 +6,6 @@ set -euo pipefail
|
||||
# Drives -#test=direnum: enumerate a long+non-ASCII directory through the
|
||||
# opendir/readdir wrappers, checking each child round-trips as UTF-8 (#133,#630).
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=direnum "$dir" | grep -q "direnum:.*OK"
|
||||
|
||||
@@ -28,8 +28,7 @@ case "$bin" in
|
||||
esac
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_doitlog.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
site="$tmp/site"
|
||||
out="$tmp/out"
|
||||
|
||||
@@ -8,8 +8,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_filelist.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
echo '<html><body>hi</body></html>' >"$tmp/index.html"
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ MINGW* | MSYS* | CYGWIN*) exit 77 ;;
|
||||
esac
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# A few-hundred-char file path (kept well under the URL length limit) makes the
|
||||
# {path} field long.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
rc=0
|
||||
out=$(httrack -#test=fsize "$dir") || rc=$?
|
||||
|
||||
@@ -10,8 +10,7 @@ echo "$out"
|
||||
test "$out" == "growsize self-test OK"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_growsize.XXXXXX")
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
echo '<html><body>hi</body></html>' >"$tmp/index.html"
|
||||
printf -- '-*/zzmarker*\n' >"$tmp/rules.txt"
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Runs the lastchar self-test (#770, #781), then keeps the idiom it replaced
|
||||
# from coming back: x[strlen(x) - 1] and x + strlen(x) - 1 both land one byte
|
||||
# before the buffer when x is empty, and the guard is never where they are.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
out=$(httrack -#test=lastchar) || {
|
||||
echo "httrack -#test=lastchar exited non-zero: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Exact-match the success line so a renamed/removed test can't pass silently.
|
||||
test "$out" = "lastchar self-test: OK" || {
|
||||
echo "expected 'lastchar self-test: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The self-test only covers the helpers, so guard the class at the source: the
|
||||
# lone survivor is inside a comment block in htsname.c. coucal is vendored, so
|
||||
# its idioms are not ours to police.
|
||||
hits=$(command grep -rn 'strlen([^)]*) *- *1\]' "$top_srcdir/src" \
|
||||
--include='*.c' --include='*.h' --exclude-dir=coucal |
|
||||
grep -v 'htsname\.c:.*save\[strlen(save)-1\]' |
|
||||
grep -v 'Replaces s\[strlen(s) - 1\]' |
|
||||
grep -v 'The x\[strlen(x) - 1\] class' || true)
|
||||
|
||||
test -z "$hits" || {
|
||||
echo "x[strlen(x) - 1] reintroduced; use hts_lastchar/hts_striplastchar/hts_choplastchar:" >&2
|
||||
echo "$hits" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Same for the pointer spelling. The three survivors sit in commented-out
|
||||
# blocks in htsname.c; the rest are this class being named, plus the self-test's
|
||||
# own deliberate underflow.
|
||||
hits=$(command grep -rnE '\+ *\(?(int|size_t)?\)? *strlen *\([^)]*\) *- *1' \
|
||||
"$top_srcdir/src" --include='*.c' --include='*.h' --exclude-dir=coucal |
|
||||
grep -v 'htsname\.c:.*char\* a=\(fil+strlen(fil)\|save+strlen(save)\)-1;' |
|
||||
grep -v 'htsselftest\.c:.*pointer spelling x + strlen(x) - 1' |
|
||||
grep -v 'htsselftest\.c:.*\*(s + strlen(s) - 1) =' || true)
|
||||
|
||||
test -z "$hits" || {
|
||||
echo "x + strlen(x) - 1 reintroduced; use hts_lastcharptr:" >&2
|
||||
echo "$hits" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# url_savename() has nine of these sites and reaches all of them with an empty
|
||||
# "fil", which "?x=1" produces. Names are asserted loosely, since the point is
|
||||
# the run: on unfixed code the sanitized CI leg aborts inside htsname.c.
|
||||
for args in "?x=1 text/plain" "?x=1 text/html" "?x=1 text/html type=4"; do
|
||||
# shellcheck disable=SC2086
|
||||
name=$(httrack -#test=savename $args) || {
|
||||
echo "httrack -#test=savename $args exited non-zero: $name" >&2
|
||||
exit 1
|
||||
}
|
||||
case "$name" in
|
||||
"savename: "?*) ;;
|
||||
*)
|
||||
echo "-#test=savename $args produced no name: $name" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# a log callback consumed the va_list the log file's vfprintf() still needed (#801).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
httrack -O /dev/null -#test=logcallback | grep -q "logcallback self-test OK"
|
||||
@@ -6,6 +6,6 @@ set -euo pipefail
|
||||
# Drives -#test=longpath: a >MAX_PATH round trip exercising hts_pathToUCS2's
|
||||
# \\?\ prefixing on Windows (#133); a positive control on POSIX.
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=longpath "$dir" | grep -q "longpath:.*OK"
|
||||
|
||||
@@ -6,7 +6,7 @@ set -euo pipefail
|
||||
# hts_finish_makeindex writes the footer and gates the refresh meta on a single
|
||||
# first link (guards the macro->function extraction).
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=makeindex "$dir" run |
|
||||
grep -q "makeindex self-test OK"
|
||||
|
||||
@@ -7,6 +7,6 @@ set -euo pipefail
|
||||
# (>MAX_PATH) and non-ASCII, exercising the mirror I/O wrappers the engine's
|
||||
# raw file ops now route to on Windows (#133, #630). Positive control on POSIX.
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=mirrorio "$dir" | grep -q "mirrorio:.*OK"
|
||||
|
||||
@@ -7,8 +7,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_parse.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# a minimal valid 1x1 GIF, reused for every referenced asset
|
||||
gif() {
|
||||
|
||||
@@ -30,8 +30,7 @@ case "$bin" in
|
||||
esac
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_rcfile.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# HTS_HTTRACKRC is ".httrackrc" on POSIX but "httrackrc" on Windows: write both,
|
||||
# each platform reads the one it knows.
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
out=$(httrack -#test=reconcile "$dir")
|
||||
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Drives -#test=renameover: hts_rename_over() must replace an existing dst, and
|
||||
# must leave dst alone when the rename failed for a reason removing dst cannot
|
||||
# fix (#779). The selftest prints the regime it detected; pin it per platform so
|
||||
# a leg cannot pass having tested the other half.
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
|
||||
case "$(uname -s)" in
|
||||
MINGW* | MSYS_NT*) want=fallback ;; # native rename() refuses an existing target
|
||||
*) want=clobber ;;
|
||||
esac
|
||||
|
||||
out=$(httrack -O /dev/null -#test=renameover "$dir")
|
||||
echo "$out" | grep -q "renameover: OK"
|
||||
echo "$out" | grep -q "renameover: regime $want"
|
||||
|
||||
if [ "$(uname -s)" != "Linux" ]; then
|
||||
echo "renameover: LD_PRELOAD interposition is Linux-only here, skipping"
|
||||
exit 0
|
||||
fi
|
||||
# A --disable-shared build has nothing to preload; anything else missing is a
|
||||
# build problem, not a skip, or the interposed legs would pass vacuously.
|
||||
if [ ! -r "${RENAMEFAIL_LA:-}" ]; then
|
||||
echo "renameover: ${RENAMEFAIL_LA:-\$RENAMEFAIL_LA} was not built" >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q "^dlname=''" "$RENAMEFAIL_LA"; then
|
||||
echo "renameover: static-only build, skipping the interposed legs"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -r "${RENAMEFAIL_LIB:-}" ]; then
|
||||
echo "renameover: ${RENAMEFAIL_LIB:-\$RENAMEFAIL_LIB} was not built" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# An LD_PRELOAD library loads ahead of the executable's own libasan, which ASan
|
||||
# refuses by default. The shim allocates nothing, so the ordering is harmless.
|
||||
export ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}verify_asan_link_order=0"
|
||||
|
||||
# The unlink fallback is dead code on POSIX, so borrow Windows' rename().
|
||||
out=$(LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
|
||||
-#test=renameover "$dir")
|
||||
echo "$out" | grep -q "renameover: OK"
|
||||
echo "$out" | grep -q "renameover: regime fallback"
|
||||
|
||||
# A source another process holds fails with EACCES, which dst had no part in.
|
||||
out=$(RENAMEFAIL_MODE=locked LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
|
||||
-#test=renameover "$dir")
|
||||
echo "$out" | grep -q "renameover: OK"
|
||||
echo "$out" | grep -q "renameover: regime refused"
|
||||
@@ -12,7 +12,7 @@ httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
# scratch dir: body= and cached= write temp files (st-savename-body.tmp, hts-cache/)
|
||||
scratch=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$scratch"' EXIT
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
run() {
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=structcheck "$dir" |
|
||||
grep -q "structcheck self-test OK"
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_threadwait_st.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmpdir"' EXIT
|
||||
trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# No pipe into grep: SIGPIPE would mask a failing exit status.
|
||||
expect_ok() {
|
||||
|
||||
@@ -6,7 +6,7 @@ set -euo pipefail
|
||||
# hts_buildtopindex takes a system-charset path but verif_backblue below it
|
||||
# expects utf-8, mangling a non-ASCII project dir on Windows (#216, #217).
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=topindex "$dir" run |
|
||||
grep -q "topindex self-test OK"
|
||||
|
||||
@@ -5,7 +5,7 @@ set -euo pipefail
|
||||
|
||||
# Accept-Encoding (#450): advertise gzip+deflate; decode gzip/zlib/raw-deflate.
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=acceptencoding "$dir" run |
|
||||
grep -q "acceptencoding self-test OK"
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# the smashed-header case logs expected "Corrupted cache entry" warnings on
|
||||
# stdout; the verdict is the last line
|
||||
|
||||
@@ -27,7 +27,7 @@ test -e "$fixture/hts-cache/new.zip" || {
|
||||
}
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# Read against a private copy so the source tree is never touched (a read
|
||||
# session does not write, but copying keeps the test hermetic). Create the dir
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# the refusal errors land on stdout (no log file); pin them and the verdict
|
||||
out=$(httrack -#test=cache-legacy "$dir" 2>/dev/null)
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
out=$(httrack -#test=cache-writefail "$dir")
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# The working directory is a required argument; without it the test prints a
|
||||
# usage line to stderr and returns non-zero.
|
||||
|
||||
@@ -5,7 +5,7 @@ set -euo pipefail
|
||||
|
||||
# brotli/zstd decode, unknown codings, and the decoded-size budget.
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=contentcodings "$dir" run |
|
||||
grep -q "contentcodings self-test OK"
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
out=$(httrack -#test=zip-repair-shift "$dir")
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ set -euo pipefail
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$scratch"' EXIT
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
name() {
|
||||
|
||||
@@ -9,7 +9,7 @@ set -euo pipefail
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$scratch"' EXIT
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc-cdx "$scratch/")
|
||||
echo "$out"
|
||||
|
||||
@@ -16,7 +16,7 @@ if ! "$httrack_bin" -#test 2>&1 | grep -q '^ warc-wacz'; then
|
||||
fi
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$scratch"' EXIT
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
out=$("$httrack_bin" -O /dev/null -#test=warc-wacz "$scratch/")
|
||||
echo "$out"
|
||||
|
||||
@@ -10,7 +10,7 @@ set -euo pipefail
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$scratch"' EXIT
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
for t in warc warc-trunc warc-ftp warc-rotate warc-verbatim; do
|
||||
out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")
|
||||
|
||||
@@ -29,7 +29,7 @@ command -v httrack >/dev/null 2>&1 || {
|
||||
tmp=$(mktemp) || exit 1
|
||||
committed_clean=$(mktemp) || exit 1
|
||||
generated_clean=$(mktemp) || exit 1
|
||||
trap 'set +e; rm -f "$tmp" "$committed_clean" "$generated_clean"' EXIT
|
||||
trap 'rm -f "$tmp" "$committed_clean" "$generated_clean"' EXIT
|
||||
|
||||
README="$top_srcdir/README" bash "$gen" httrack >"$tmp" 2>/dev/null || {
|
||||
echo "makeman.sh failed" >&2
|
||||
|
||||
@@ -14,7 +14,7 @@ set -euo pipefail
|
||||
|
||||
site=$(mktemp -d)
|
||||
out=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$site" "$out"' EXIT
|
||||
trap 'rm -rf "$site" "$out"' EXIT
|
||||
|
||||
cat >"$site/index.html" <<EOF
|
||||
<a href="a.html">a</a> <a href="sub/b.html">b</a>
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The end-of-mirror purge read old.lst with linput(fp, line, 1000), which caps a
|
||||
# chunk at 999 bytes. filenote() writes "[<save path>]", so a 998-byte save path
|
||||
# makes a 1000-byte line that comes back as 999 bytes plus a one-byte "]" tail.
|
||||
# For that tail `line + 1` is empty, and with no -O so is path_html, so the purge
|
||||
# indexed file[-1] (#770). Needs no -O, hence not local-crawl.sh.
|
||||
#
|
||||
# The OOB write lands in stack padding, so only the sanitized CI leg fails on
|
||||
# unfixed code; the length assertion below is what keeps this from passing
|
||||
# vacuously if the save path ever stops landing on the chunk boundary.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
# python3 is required; mirror check-network.sh's skip-with-77 convention.
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
# Resolve httrack now: the harness prepends a RELATIVE dir to PATH, so the cd
|
||||
# below would otherwise silently fall through to an installed copy.
|
||||
httrack_bin=$(command -v httrack)
|
||||
test -n "$httrack_bin" || {
|
||||
echo "httrack not on PATH" >&2
|
||||
exit 1
|
||||
}
|
||||
httrack_bin=$(cd "$(dirname "$httrack_bin")" && pwd)/$(basename "$httrack_bin")
|
||||
|
||||
work=$(mktemp -d)
|
||||
trap 'set +e; kill -9 "${spid:-}" 2>/dev/null || true; rm -rf "$work"' EXIT
|
||||
|
||||
mkdir -p "$work/root" "$work/proj"
|
||||
|
||||
"$python" "$top_srcdir/tests/local-server.py" --root "$work/root" \
|
||||
>"$work/server.out" 2>"$work/server.err" &
|
||||
spid=$!
|
||||
|
||||
port=
|
||||
for _ in $(seq 1 100); do
|
||||
port=$(sed -n 's/^PORT \([0-9]*\)$/\1/p' "$work/server.out" 2>/dev/null || true)
|
||||
test -n "$port" && break
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "local-server.py did not announce a port" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Save path is "127.0.0.1_<port>/" + the URL path; target 998 bytes total.
|
||||
prefix="127.0.0.1_${port}/"
|
||||
need=$((998 - ${#prefix}))
|
||||
|
||||
# macOS caps PATH_MAX at 1024, so neither the docroot nor the mirror can hold a
|
||||
# 998-byte path, and httrack's own HTS_MAX_PATH_LEN clamps below it too. The
|
||||
# trigger is simply not reachable there.
|
||||
pathmax=$(getconf PATH_MAX / 2>/dev/null || echo 4096)
|
||||
test "$pathmax" -ge $((need + ${#work} + 128)) || {
|
||||
echo "PATH_MAX $pathmax too small for a $need-byte path; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
# Same on Win32, where MAX_PATH clamps the save path; getconf answers for the
|
||||
# MSYS emulation, not the Win32 API httrack calls, so it cannot see this.
|
||||
case "$(uname -s)" in
|
||||
MINGW* | MSYS* | CYGWIN*)
|
||||
echo "Win32 MAX_PATH cannot hold a $need-byte save path; skipping" >&2
|
||||
exit 77
|
||||
;;
|
||||
esac
|
||||
|
||||
urlpath=""
|
||||
while test $((need - ${#urlpath})) -gt 60; do
|
||||
urlpath="${urlpath}$(printf 'd%02d' ${#urlpath})$(printf 'a%.0s' $(seq 1 46))/"
|
||||
done
|
||||
urlpath="${urlpath}$(printf 'f%.0s' $(seq 1 $((need - ${#urlpath} - 5)))).html"
|
||||
test ${#urlpath} -eq "$need"
|
||||
|
||||
mkdir -p "$work/root/$(dirname "$urlpath")"
|
||||
printf '<html><body>leaf</body></html>\n' >"$work/root/$urlpath"
|
||||
printf '<html><body><a href="/%s">x</a></body></html>\n' "$urlpath" \
|
||||
>"$work/root/index.html"
|
||||
|
||||
cd "$work/proj" # no -O: opt->path_html stays empty
|
||||
|
||||
"$httrack_bin" "http://127.0.0.1:$port/" -q -%v0 -s0 -r3 >"$work/crawl1.log" 2>&1 || true
|
||||
|
||||
lst=$work/proj/hts-cache/new.lst
|
||||
test -f "$lst" || {
|
||||
echo "pass 1 produced no new.lst" >&2
|
||||
exit 1
|
||||
}
|
||||
# Arms the test: without a line of exactly 1000 bytes linput() never splits and
|
||||
# the purge never sees a one-byte tail.
|
||||
awk 'length == 1000 { found = 1 } END { exit !found }' "$lst" || {
|
||||
echo "no 1000-byte line in new.lst; the chunk split is not exercised" >&2
|
||||
awk '{ print length }' "$lst" | sort -n | tail -3 >&2
|
||||
echo "--- urlpath len ${#urlpath}, prefix $prefix" >&2
|
||||
tail -20 "$work/crawl1.log" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Pass 2 renames new.lst to old.lst and runs it through the purge loop.
|
||||
"$httrack_bin" "http://127.0.0.1:$port/" -q -%v0 -s0 -r3 >"$work/crawl2.log" 2>&1
|
||||
|
||||
test -s "$work/proj/$prefix$urlpath" || {
|
||||
echo "the mirrored leaf did not survive the purge" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A .bak left behind by a killed run must not disable the #77 re-fetch backup
|
||||
# (#758): a leftover file (slow.bin.bak) has to be clobbered so the -M abort can
|
||||
# still restore the pass-1 copy, and a leftover the engine cannot clobber
|
||||
# (fast.bin.bak, planted as a directory) has to be reported instead of silently
|
||||
# truncating with no safety net. Only the Windows leg arms the first half:
|
||||
# POSIX rename() clobbers on its own.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" \
|
||||
--plant-file bigtrunc/slow.bin.bak \
|
||||
--plant-dir bigtrunc/fast.bin.bak \
|
||||
--rerun-args '--update -M400000' \
|
||||
--log-found 'More than 400000 bytes have been transferred.. giving up' \
|
||||
--log-found 'replacing leftover backup .*slow\.bin\.bak' \
|
||||
--log-found 'could not back up .*fast\.bin' \
|
||||
--log-not-found 'could not back up .*slow\.bin' \
|
||||
--log-not-found 'could not restore' \
|
||||
--found bigtrunc/slow.bin \
|
||||
--found bigtrunc/fast.bin \
|
||||
--file-min-bytes bigtrunc/slow.bin 655360 \
|
||||
--file-not-matches bigtrunc/slow.bin 'stale-leftover' \
|
||||
httrack 'BASEURL/bigtrunc/index.html' -c2
|
||||
@@ -1,164 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An FTP re-fetch used to truncate the mirror before the transfer, so a failed
|
||||
# one destroyed the previous copy (#771). keep.bin is cut short and empty.bin
|
||||
# gets nothing; both must keep their old bytes. stay.bin takes the same path and
|
||||
# completes, so a fix that merely stopped writing would fail here.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
command -v httrack >/dev/null || {
|
||||
echo "could not find httrack" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
server=$(nativepath "${testdir}/ftp-server.py")
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
|
||||
serverpid=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
|
||||
root="${tmpdir}/root"
|
||||
out="${tmpdir}/crawl"
|
||||
mode="${tmpdir}/mode"
|
||||
report="${out}/hts-changes.json"
|
||||
mkdir -p "$root" "$out"
|
||||
|
||||
# The two generations differ in length, or a re-fetch that resumed at EOF
|
||||
# instead of truncating would leave the same bytes and pass for the wrong reason.
|
||||
write_bodies() {
|
||||
local gen="$1" fill="$2" name
|
||||
for name in keep empty stay; do
|
||||
"$python" -c 'import sys; sys.stdout.buffer.write(
|
||||
("%s-FTP-%s " % (sys.argv[1].upper(), sys.argv[2])).encode()
|
||||
+ sys.argv[2][-1].encode() * int(sys.argv[3]))' "$name" "$gen" "$fill" \
|
||||
>"${root}/${name}.bin"
|
||||
done
|
||||
}
|
||||
write_bodies V1 4096
|
||||
: >"$mode"
|
||||
|
||||
serverlog="${tmpdir}/server.out"
|
||||
"$python" "$server" --root "$(nativepath "$root")" \
|
||||
--mode-file "$(nativepath "$mode")" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 300); do
|
||||
line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "ftp server exited early: $(cat "$serverlog")" >&2
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover ftp server port: $(cat "$serverlog")" >&2
|
||||
exit 1
|
||||
}
|
||||
host="127.0.0.1_${port}"
|
||||
urls=()
|
||||
for name in keep empty stay; do
|
||||
urls+=("ftp://127.0.0.1:${port}/${name}.bin")
|
||||
done
|
||||
# -c1: a parallel FTP crawl loses whole transfers to a separate, older bug in
|
||||
# the backlog slot swap (#797), which would flake this test on a loaded runner.
|
||||
common=(--quiet --disable-security-limits --robots=0 --timeout=20 --max-time=120
|
||||
--retries=1 -c1 --changes)
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
test ! -f "$report" || cat "$report" >&2
|
||||
exit 1
|
||||
}
|
||||
ok() { echo "OK: $*"; }
|
||||
size_of() { wc -c <"$1" | tr -d '[:space:]'; }
|
||||
# Files listed under a report bucket, one "file:size" per line.
|
||||
listed() {
|
||||
"$python" - "$report" "$1" <<'EOF' | tr -d '\r'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as fp:
|
||||
report = json.load(fp)
|
||||
for entry in report[sys.argv[2]]:
|
||||
print("%s:%s" % (entry["file"], entry.get("size", "none")))
|
||||
EOF
|
||||
}
|
||||
|
||||
# --- pass 1: a healthy mirror ------------------------------------------------
|
||||
# Bounded like every local-crawl.sh pass: a wedged crawl must fail the test, not
|
||||
# hang the job until CI cancels it and discards the log (#796).
|
||||
run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" \
|
||||
>"${tmpdir}/log1" 2>&1
|
||||
for name in keep empty stay; do
|
||||
test -s "${out}/${host}/${name}.bin" || fail "pass 1 did not mirror ${name}.bin"
|
||||
done
|
||||
mkdir "${tmpdir}/snap"
|
||||
cp "${out}/${host}"/*.bin "${tmpdir}/snap/"
|
||||
ok "pass 1 mirrored three files"
|
||||
|
||||
# --- pass 2: the same three re-fetched, two of them failing -------------------
|
||||
# norest throughout: an accepted REST sends the client down the append path,
|
||||
# which is not the one that truncates.
|
||||
write_bodies V2 6000
|
||||
cat >"$mode" <<'EOF'
|
||||
/keep.bin truncate norest
|
||||
/empty.bin empty norest
|
||||
/stay.bin norest
|
||||
EOF
|
||||
run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" --update \
|
||||
>"${tmpdir}/log2" 2>&1
|
||||
|
||||
# Both failures have to be the transfer dying mid-body: a connect or lookup
|
||||
# failure never opens the file, and everything below would pass vacuously.
|
||||
for name in keep empty; do
|
||||
grep -aqE "Error:.*\"FTP file incomplete\".*${name}\.bin" "${out}/hts-log.txt" ||
|
||||
fail "${name}.bin did not fail during the transfer"
|
||||
done
|
||||
ok "both re-fetches reached the body and failed there"
|
||||
|
||||
for name in keep empty; do
|
||||
cmp -s "${out}/${host}/${name}.bin" "${tmpdir}/snap/${name}.bin" ||
|
||||
fail "${name}.bin differs from the copy pass 1 left ($(size_of \
|
||||
"${out}/${host}/${name}.bin") bytes now)"
|
||||
done
|
||||
ok "a failed FTP transfer left the previously mirrored bytes alone"
|
||||
|
||||
grep -aq "FTP-V2 " "${out}/${host}/stay.bin" ||
|
||||
fail "stay.bin was not refreshed; the re-fetch writes nothing at all now"
|
||||
ok "a successful FTP re-fetch still replaces the file"
|
||||
|
||||
leftover=$(find "${out}/${host}" -name '*.bak')
|
||||
test -z "$leftover" || fail "backup left behind: ${leftover}"
|
||||
ok "no backup temporary survived the pass"
|
||||
|
||||
# --purge-old is on by default: a kept file the run never replaced must still be
|
||||
# in new.lst, or it is deleted as gone (#746).
|
||||
for name in keep empty stay; do
|
||||
test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was purged"
|
||||
done
|
||||
ok "the kept copies survived the update purge"
|
||||
|
||||
# --- the change report is the mirror's own account of the pass ----------------
|
||||
test -s "$report" || fail "pass 2 wrote no ${report}"
|
||||
test -z "$(listed gone)" || fail "the report calls something gone: $(listed gone)"
|
||||
for name in keep empty; do
|
||||
listed unchanged | grep -qx "${host}/${name}.bin:$(size_of "${tmpdir}/snap/${name}.bin")" ||
|
||||
fail "${name}.bin is not reported unchanged at its previous size"
|
||||
if listed changed | grep -q "^${host}/${name}.bin:"; then
|
||||
fail "${name}.bin is reported changed as well"
|
||||
fi
|
||||
done
|
||||
listed changed | grep -qx "${host}/stay.bin:$(size_of "${out}/${host}/stay.bin")" ||
|
||||
fail "stay.bin is not reported changed"
|
||||
ok "the change report calls the kept files unchanged and the refreshed one changed"
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Issue #773: a test must fail only when an assertion fails. Under "set -e" a
|
||||
# failing command in an EXIT trap becomes the script's exit status, so a
|
||||
# teardown hiccup turned a test whose assertions all passed into a red CI leg.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# run <trap-body> <script-body>: exit status and stderr of a subject script.
|
||||
run() {
|
||||
{
|
||||
echo "set -euo pipefail"
|
||||
# printf, so the scanner below does not read this as a trap of our own
|
||||
printf 'trap %s EXIT\n' "$1"
|
||||
echo "$2"
|
||||
} >"$tmp/subject"
|
||||
rc=0
|
||||
err=$(bash "$tmp/subject" 2>&1 >/dev/null) || rc=$?
|
||||
}
|
||||
|
||||
# 1. the shape in the tree: teardown fails, every assertion passed -> pass
|
||||
run "'set +e; false'" "true"
|
||||
test "$rc" -eq 0 || fail "guarded teardown failure exited $rc, want 0"
|
||||
|
||||
# 2. teeth for case 1: unguarded, the same teardown is what went red
|
||||
run "'false'" "true"
|
||||
test "$rc" -eq 1 || fail "unguarded teardown failure exited $rc, want 1"
|
||||
|
||||
# 3. the guard must not swallow a real failure, nor a skip
|
||||
run "'set +e; false'" "exit 1"
|
||||
test "$rc" -eq 1 || fail "assertion failure exited $rc, want 1"
|
||||
run "'set +e; false'" "exit 77"
|
||||
test "$rc" -eq 77 || fail "skip exited $rc, want 77"
|
||||
|
||||
# 4. not failing must not mean going quiet: teardown still reports why
|
||||
run "'set +e; rm /nonexistent-teardown-probe'" "true"
|
||||
test "$rc" -eq 0 || fail "guarded teardown failure exited $rc, want 0"
|
||||
case "$err" in
|
||||
*nonexistent-teardown-probe*) ;;
|
||||
*) fail "teardown error was silenced (stderr: '$err')" ;;
|
||||
esac
|
||||
echo "OK: teardown cannot decide the verdict, and still reports"
|
||||
|
||||
# 5. no test may reintroduce the shape case 2 pins. Two spellings neutralise
|
||||
# errexit for teardown: "set +e" first, or "|| true" over the whole body.
|
||||
scanned=0
|
||||
for f in "$top_srcdir"/tests/*.test "$top_srcdir"/tests/*.sh; do
|
||||
test -r "$f" || continue
|
||||
scanned=$((scanned + 1))
|
||||
done
|
||||
# an unexpanded glob would report no offender having read nothing
|
||||
test "$scanned" -ge 50 || fail "scanned $scanned scripts in $top_srcdir/tests, want the whole suite"
|
||||
|
||||
bad=$(awk '
|
||||
FNR == 1 { errexit = 0 }
|
||||
/^[[:space:]]*set[[:space:]]+(-[a-z]*e|-o[[:space:]]+errexit)/ { errexit = 1 }
|
||||
!errexit { next }
|
||||
!/^[[:space:]]*trap[[:space:]]+.*[[:space:]]EXIT([[:space:]]|;|$)/ { next }
|
||||
{
|
||||
guarded = /^[[:space:]]*trap[[:space:]]+.set \+e; / || /\|\|[[:space:]]+true.[[:space:]]+EXIT/
|
||||
if (!guarded)
|
||||
print FILENAME ":" FNR " unguarded"
|
||||
# "set +e" on a trap shared with a signal leaves errexit off for the
|
||||
# rest of the run, so a torn-down test could still report success
|
||||
else if (/^[[:space:]]*trap[[:space:]]+.set \+e; / && /[[:space:]]EXIT[[:space:]]+[A-Z]/)
|
||||
print FILENAME ":" FNR " set +e shared with a signal trap"
|
||||
}
|
||||
' "$top_srcdir"/tests/*.test "$top_srcdir"/tests/*.sh)
|
||||
test -z "$bad" || fail "EXIT trap can decide the verdict:
|
||||
$bad"
|
||||
echo "OK: every EXIT trap under set -e runs teardown with errexit off"
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A URL past the header-format buffer must still produce a record: the archive
|
||||
# used to lose it whole, with no error and an exit status of 0 (#785).
|
||||
|
||||
set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_longurl.XXXXXX")
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
|
||||
httrack -O "$tmp/out" "-#test=warc-longurl" "$tmp"
|
||||
@@ -1,141 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The harness timeout (tests/test-timeout.sh) must name the test it killed, say
|
||||
# what was still running, and exit 124 -- otherwise a wedge only ever shows up as
|
||||
# a cancelled CI step, which keeps no log and no artifact.
|
||||
|
||||
set -eu
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
driver="${testdir}/test-timeout.sh"
|
||||
test -r "$driver" || {
|
||||
echo "no test-timeout.sh next to $0" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
test -z "${out:-}" || sed 's/^/ | /' <"$out" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_sutmo.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
out="$tmp/out"
|
||||
|
||||
# --- a hanging test is killed, named, and reported 124 ----------------------
|
||||
cat >"$tmp/90_wedged.test" <<'EOF'
|
||||
echo "marker: the wedged test started"
|
||||
sleep 300
|
||||
EOF
|
||||
|
||||
start=$SECONDS
|
||||
rc=0
|
||||
HTTRACK_TEST_TIMEOUT=5 bash "$driver" "$tmp/90_wedged.test" >"$out" 2>&1 || rc=$?
|
||||
elapsed=$((SECONDS - start))
|
||||
|
||||
test "$rc" -eq 124 || fail "wedged test reported $rc, want 124"
|
||||
# Never before the budget, or a slow-but-healthy test would be killed too.
|
||||
test "$elapsed" -ge 5 || fail "the guard fired early (${elapsed}s of a 5s budget)"
|
||||
test "$elapsed" -lt 30 || fail "the guard fired late (${elapsed}s)"
|
||||
grep -q 'marker: the wedged test started' "$out" || fail "the test's own output was lost"
|
||||
# The header, not a bare name: the process list quotes the test's path too, so a
|
||||
# wrapper that named nothing would still match that.
|
||||
grep -q '^===== TIMEOUT: 90_wedged.test exceeded' "$out" ||
|
||||
fail "the diagnostics do not name the test"
|
||||
grep -q "own process tree" "$out" || fail "no process list in the diagnostics"
|
||||
|
||||
# The budget is read, not hard-coded: well under it, the same shape survives.
|
||||
printf 'sleep 3\necho "slow but healthy"\n' >"$tmp/92_slow.test"
|
||||
rc=0
|
||||
HTTRACK_TEST_TIMEOUT=45 bash "$driver" "$tmp/92_slow.test" >"$out" 2>&1 || rc=$?
|
||||
test "$rc" -eq 0 || fail "a healthy 3s test under a 45s budget reported $rc"
|
||||
grep -q 'slow but healthy' "$out" || fail "healthy test output lost"
|
||||
! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a healthy test"
|
||||
|
||||
# The killed tree must really be gone, or the next test inherits its ports.
|
||||
sleep 1
|
||||
! grep -q "$tmp/90_wedged.test" <<<"$(ps -A -o args 2>/dev/null)" ||
|
||||
fail "the wedged test survived the guard"
|
||||
|
||||
# --- exit status and output of a healthy test pass straight through ----------
|
||||
# 77 is automake's SKIP and 99 its hard error, so both have to survive the wrapper.
|
||||
for want in 0 1 77 99; do
|
||||
printf 'echo "ran with %s"\nexit %s\n' "$want" "$want" >"$tmp/91_plain.test"
|
||||
rc=0
|
||||
HTTRACK_TEST_TIMEOUT=60 bash "$driver" "$tmp/91_plain.test" >"$out" 2>&1 || rc=$?
|
||||
test "$rc" -eq "$want" || fail "exit $want came back as $rc"
|
||||
grep -q "ran with $want" "$out" || fail "output lost for exit $want"
|
||||
! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a test that exited at once"
|
||||
done
|
||||
|
||||
# A budget of 0 disables the guard: the test still runs, and is not killed. Its
|
||||
# own file, so the assertion cannot ride on whatever the loop above left behind.
|
||||
printf 'echo "guard off"\nexit 3\n' >"$tmp/93_off.test"
|
||||
rc=0
|
||||
HTTRACK_TEST_TIMEOUT=0 bash "$driver" "$tmp/93_off.test" >"$out" 2>&1 || rc=$?
|
||||
test "$rc" -eq 3 || fail "disabled guard changed the exit status ($rc)"
|
||||
grep -q 'guard off' "$out" || fail "output lost with the guard disabled"
|
||||
|
||||
# --- a wedged crawl yields a symbolized engine stack ------------------------
|
||||
# The whole point of the dump: name the frame the engine is stuck in. Windows has
|
||||
# neither half (MSYS signals do not reach a native httrack.exe, and that build has
|
||||
# no backtrace()), so only assert it where the machinery exists.
|
||||
if is_windows; then
|
||||
echo "suite timeout OK (no stack capture on Windows)"
|
||||
exit 0
|
||||
fi
|
||||
command -v httrack >/dev/null || {
|
||||
echo "SKIP: no httrack on PATH for the stack half" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
# /watchdog/stall never answers, so the crawl wedges for real. CRAWL_DEADLINE is
|
||||
# raised past our budget so the guard under test is what fires, not local-crawl's.
|
||||
cat >"$tmp/92_stalled.test" <<EOF
|
||||
CRAWL_DEADLINE=600 bash "${testdir}/local-crawl.sh" --errors 0 \\
|
||||
httrack 'BASEURL/watchdog/stall'
|
||||
EOF
|
||||
|
||||
rc=0
|
||||
HTTRACK_TEST_TIMEOUT=20 bash "$driver" "$tmp/92_stalled.test" >"$out" 2>&1 || rc=$?
|
||||
|
||||
if grep -qE 'python3 not found|could not discover server port' "$out"; then
|
||||
echo "SKIP: local crawl prerequisites missing" >&2
|
||||
exit 77
|
||||
fi
|
||||
test "$rc" -eq 124 || fail "stalled crawl reported $rc, want 124"
|
||||
|
||||
if grep -q 'No stack trace available on this OS' "$out"; then
|
||||
echo "suite timeout OK (no backtrace() on this platform)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Each platform's mechanism has its own marker, and the marker is what proves the
|
||||
# whole chain connected up rather than the dump merely running.
|
||||
case "$(uname -s)" in
|
||||
Linux)
|
||||
# sig_fatal writes to the engine's own stderr, so this reaching our log also
|
||||
# proves the crawl-log salvage worked.
|
||||
grep -q '^Caught signal ' "$out" || fail "no engine backtrace was salvaged"
|
||||
;;
|
||||
Darwin)
|
||||
grep -q '^Call graph:' "$out" || fail "sample(1) produced no call graph"
|
||||
;;
|
||||
*)
|
||||
grep -q 'no stack mechanism known' "$out" ||
|
||||
fail "unknown platform reported neither a stack nor its absence"
|
||||
echo "suite timeout OK (no stack mechanism on $(uname -s))"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
# A stalled crawl is necessarily inside httpmirror, so these frames must appear.
|
||||
# Matching "httrack" alone would pass on the process list above.
|
||||
grep -qE '\b(back_wait|httpmirror|hts_main_internal)\b' "$out" ||
|
||||
fail "the stack names no engine frame"
|
||||
|
||||
echo "suite timeout OK (named the test, dumped a stack, exited 124)"
|
||||
@@ -1,107 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Backward scans that start at s + strlen(s) - 1 and decrement with no lower
|
||||
# bound: the config-file right trim and the collision-rename digit scan both
|
||||
# walked off the front of a stack buffer. Neither ASan nor _FORTIFY_SOURCE sees
|
||||
# the self-test's overrun (it lands in the same frame), hence the poisoned byte.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# Resolve httrack before any cd: PATH may carry a build-relative entry.
|
||||
bin=$(command -v httrack) || {
|
||||
echo "FAIL: httrack not found on PATH" >&2
|
||||
exit 1
|
||||
}
|
||||
case "$bin" in
|
||||
/*) ;;
|
||||
*) bin="$(cd "$(dirname "$bin")" && pwd)/$(basename "$bin")" ;;
|
||||
esac
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_rtrim.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- 1. the helper itself ---------------------------------------------------
|
||||
out=$("$bin" -#test=rtrim) || fail "httrack -#test=rtrim exited non-zero: $out"
|
||||
test "$out" = "rtrim self-test: OK" || fail "expected 'rtrim self-test: OK', got: $out"
|
||||
|
||||
# --- 2. collision rename over an all-digit name -----------------------------
|
||||
# Two pages with the same all-digit name save to 2024.html, so the second goes
|
||||
# through the -<n> rename and the digit scan has nothing but digits to walk
|
||||
# over. Driven twice, because the two routes differ in who picks the URLs.
|
||||
site="$tmp/site"
|
||||
mkdir -p "$site/a" "$site/b"
|
||||
printf '%s\n' '<html><body><a href="a/2024.html">a</a> <a href="b/2024.html">b</a></body></html>' \
|
||||
>"$site/index.html"
|
||||
echo '<html><body>A</body></html>' >"$site/a/2024.html"
|
||||
echo '<html><body>B</body></html>' >"$site/b/2024.html"
|
||||
|
||||
# -g flattens the tree, and pins depth to 0: the operator supplies both URLs.
|
||||
out1="$tmp/out1"
|
||||
mkdir -p "$out1"
|
||||
(cd "$out1" && "$bin" -g "file://$site/a/2024.html" "file://$site/b/2024.html" \
|
||||
--quiet -n >.log 2>&1) || fail "collision crawl exited non-zero: $(tail -3 "$out1/.log")"
|
||||
|
||||
test -f "$out1/2024.html" || fail "2024.html not mirrored"
|
||||
test -f "$out1/2024-2.html" || fail "colliding name was not renamed to 2024-2.html"
|
||||
|
||||
# -N "%n.%t" collides the same way with depth unrestricted, so one starting URL
|
||||
# is enough and the crawled page picks both colliding names itself.
|
||||
out3="$tmp/out3"
|
||||
mkdir -p "$out3"
|
||||
(cd "$out3" && "$bin" "file://$site/index.html" -N "%n.%t" --quiet -n >.log 2>&1) ||
|
||||
fail "userdef-savename crawl exited non-zero: $(tail -3 "$out3/.log")"
|
||||
test -f "$out3/2024.html" || fail "2024.html not mirrored under -N '%n.%t'"
|
||||
test -f "$out3/2024-2.html" ||
|
||||
fail "colliding name was not renamed under -N '%n.%t' (links followed, not argv)"
|
||||
|
||||
# The rename must keep counting, not restart: a third collision is -3.
|
||||
mkdir -p "$site/c"
|
||||
echo '<html><body>C</body></html>' >"$site/c/2024.html"
|
||||
out2="$tmp/out2"
|
||||
mkdir -p "$out2"
|
||||
(cd "$out2" && "$bin" -g "file://$site/a/2024.html" "file://$site/b/2024.html" \
|
||||
"file://$site/c/2024.html" --quiet -n >.log 2>&1) ||
|
||||
fail "three-way collision crawl exited non-zero: $(tail -3 "$out2/.log")"
|
||||
test -f "$out2/2024-3.html" || fail "third colliding name was not renamed to 2024-3.html"
|
||||
|
||||
# --- 3. a blank config line is skipped, not parsed ---------------------------
|
||||
# HTS_HTTRACKRC is ".httrackrc" on POSIX and "httrackrc" on Windows.
|
||||
rcdir="$tmp/rc"
|
||||
mkdir -p "$rcdir"
|
||||
echo '<html><body>hi</body></html>' >"$rcdir/index.html"
|
||||
printf ' \nzzz-not-an-option\n' >"$rcdir/.httrackrc"
|
||||
cp "$rcdir/.httrackrc" "$rcdir/httrackrc"
|
||||
|
||||
rc=0
|
||||
log=$( (cd "$rcdir" && "$bin" "file://$rcdir/index.html" --quiet -n 2>&1)) || rc=$?
|
||||
test "$rc" -eq 0 || fail "rc-file crawl exited $rc: $log"
|
||||
log=${log//$'\r'/} # the empty-option check below must not miss a CRLF line
|
||||
|
||||
# The line after the blank one must still be read, or this proves nothing.
|
||||
case "$log" in
|
||||
*"Unknown option: zzz-not-an-option"*) ;;
|
||||
*) fail "config file was not parsed past the blank line: $log" ;;
|
||||
esac
|
||||
# The blank line itself must not become an empty option.
|
||||
case "$log" in
|
||||
*"Unknown option: "$'\n'*) fail "blank config line was parsed as an empty option: $log" ;;
|
||||
esac
|
||||
|
||||
# --- 4. the call sites stay bounded -----------------------------------------
|
||||
# Sections 2 and 3 only go red under a sanitizer: on a plain build the byte
|
||||
# below the buffer is usually not a digit, so a reverted scan stops anyway and
|
||||
# produces the same names. Pin the two sites at the source instead.
|
||||
command grep -q 'hts_rtrim(line, HTS_REALSPACES)' "$top_srcdir/src/htsalias.c" ||
|
||||
fail "htsalias.c right trim no longer goes through hts_rtrim"
|
||||
command grep -q 'hts_rtrimlen(tempo, "0123456789")' "$top_srcdir/src/htsname.c" ||
|
||||
fail "htsname.c collision-suffix scan no longer goes through hts_rtrimlen"
|
||||
|
||||
exit 0
|
||||
@@ -31,7 +31,7 @@ cleanup() {
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup EXIT
|
||||
|
||||
# self-signed cert for the local TLS origin (httrack does not verify certs)
|
||||
openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \
|
||||
|
||||
@@ -40,8 +40,9 @@ serverpid=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
return 0
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup EXIT
|
||||
|
||||
# bind the live server to 127.0.0.1 only, so 127.0.0.2 refuses the connect
|
||||
"$python" "$server" --root "$root" --bind 127.0.0.1 >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
|
||||
|
||||
@@ -24,8 +24,7 @@ cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# OVERLAP_COUNTER gets a byte per flaky.bin request so pass 1 knows when to interrupt.
|
||||
serverlog="${tmpdir}/server.log"
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# An update run against a dead server must not destroy the cache: the no-data
|
||||
# rollback restores the previous hts-cache generation (zip caches lost it). The
|
||||
# live update pass leaves an old.zip behind, so the dead pass rotates onto an
|
||||
# existing generation instead of into empty space.
|
||||
# rollback restores the previous hts-cache generation (zip caches lost it).
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun --rerun-dead \
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun-dead \
|
||||
--found 'simple/basic.html' \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
|
||||
@@ -6,7 +6,7 @@ set -euo pipefail
|
||||
# --why: report which +/- filter rule decides for a URL, without crawling.
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap 'set +e; rm -rf "$tmpdir"' EXIT
|
||||
trap 'rm -rf "$tmpdir"' EXIT
|
||||
|
||||
why() {
|
||||
local want="$1" out
|
||||
|
||||
@@ -34,7 +34,7 @@ cleanup() {
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup EXIT
|
||||
|
||||
openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \
|
||||
-out "$tmpdir/cert.pem" -days 2 -nodes -subj "/CN=127.0.0.1" \
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
printf 'HTTP/1.1 404 Not Found Here At All\r\nContent-Type: text/html\r\nLast-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
|
||||
printf 'hello' >"$dir/body"
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$dir"' EXIT
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# The first length-prefixed field declares far more than firstline[256] holds;
|
||||
# cache_brstr must clamp the copy to the destination, not the declared length.
|
||||
|
||||
@@ -29,8 +29,9 @@ serverpid=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
return 0
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup EXIT
|
||||
|
||||
# decoy origin: it must stay silent. LOCAL_SERVER_VERBOSE logs any request it gets.
|
||||
LOCAL_SERVER_VERBOSE=1 "$python" "$server" --root "$root" --bind 127.0.0.1 \
|
||||
|
||||
@@ -29,7 +29,7 @@ cleanup() {
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup EXIT
|
||||
|
||||
# start_server <logdir> <mode>: launches a proxy+origin pair, sets $origin_port
|
||||
# and $proxy_port from its announced ephemeral ports.
|
||||
|
||||
@@ -12,7 +12,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wd.XXXXXX")
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
|
||||
@@ -27,8 +27,9 @@ serverpid=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
return 0
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup EXIT
|
||||
|
||||
# start_stall_server <tag> <mode-args...>: sets $port from the announced one.
|
||||
start_stall_server() {
|
||||
|
||||
@@ -12,7 +12,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
|
||||
TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/httrack_salv.XXXXXX")
|
||||
export TMPDIR
|
||||
trap 'set +e; rm -rf "$TMPDIR"' EXIT
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
|
||||
@@ -19,7 +19,7 @@ for f in "$eng" "$def"; do
|
||||
done
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
keys="$tmp/english.keys"
|
||||
|
||||
# Byte-wise: each file is in its own declared legacy charset, lang.def is CRLF.
|
||||
|
||||
@@ -25,11 +25,10 @@ srv=
|
||||
cleanup() {
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
log=$(mktemp)
|
||||
trap 'set +e; cleanup; rm -f "${log}"' EXIT
|
||||
trap 'cleanup; rm -f "${log}"' EXIT
|
||||
|
||||
# Ask htsserver for its rendered default base path, under the given $HOME
|
||||
# ("-" = leave $HOME unset).
|
||||
|
||||
@@ -14,8 +14,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_portsib.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmp"' EXIT
|
||||
trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# A port the old code accepted only by wrapping: 4295009395 = 2^32 + 42099 and
|
||||
# 4294967376 = 2^32 + 80 both fold to a plausible listen port. They are the
|
||||
|
||||
@@ -14,7 +14,7 @@ set -euo pipefail
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
scratch=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$scratch"' EXIT
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8}) # 128 hex chars
|
||||
|
||||
@@ -36,8 +36,7 @@ cleanup() {
|
||||
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
rm -rf "${work}" "${srvlog}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
|
||||
# ~/.httrack.ini out of it.
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A --warc crawl writes a standards-conformant WARC/1.1 archive, and a re-crawl
|
||||
# of an unchanged (all-304) site emits revisit records. The stdlib validator (no
|
||||
# warcio) is the real gate: it byte-compares the fresh page.html response body
|
||||
# against what the server served, asserts the encoding headers were stripped and
|
||||
# the payload digest matches, then checks the second pass turned the unchanged
|
||||
# assets into revisits (no full response).
|
||||
#
|
||||
# That second pass archives under a name of its own, which is also the recovery
|
||||
# the engine points at when it refuses to strand a previous archive (#759).
|
||||
# A --warc crawl writes a standards-conformant WARC/1.1 archive, and an
|
||||
# --update re-crawl of an unchanged (all-304) site emits revisit records.
|
||||
# The stdlib validator (no warcio) is the real gate: it byte-compares the
|
||||
# fresh page.html response body against what the server served, asserts the
|
||||
# encoding headers were stripped and the payload digest matches, then checks
|
||||
# the update pass turned the unchanged assets into revisits (no full response).
|
||||
|
||||
set -eu
|
||||
|
||||
@@ -17,10 +14,8 @@ set -eu
|
||||
# page.html body served by tests/local-server.py (route_mini304_page).
|
||||
export WARC_VALIDATE_BODY="page.html=3c68746d6c3e3c626f64793e74696e7920636163686561626c6520706167653c2f626f64793e3c2f68746d6c3e0a"
|
||||
export WARC_VALIDATE_NORESP="index.html page.html"
|
||||
export WARC_VALIDATE_UPDATE="warc-update.warc.gz"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --warc-validate \
|
||||
--rerun-args '--warc-file warc-update' \
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun --warc-validate \
|
||||
--log-found 'no files updated' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out
|
||||
|
||||
@@ -6,10 +6,9 @@
|
||||
# the datapackage-digest chain (plus py-wacz/pywb when importable). The crawl
|
||||
# skips cleanly on a build without OpenSSL (no conformant SHA-256 -> no package).
|
||||
#
|
||||
# The cacheless (-C0) second pass repackages over the .wacz the first one left:
|
||||
# the clobber, not the happy rename, is what breaks when a platform refuses to
|
||||
# rename onto an existing file (#726). A cache-served pass keeps the previous
|
||||
# package instead, which is 97_local-warc-update-keep.
|
||||
# The cacheless second pass repackages over the .wacz the first one left: the
|
||||
# clobber, not the happy rename, is what breaks when a platform refuses to
|
||||
# rename onto an existing file (#726).
|
||||
|
||||
set -eu
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ set -euo pipefail
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
scratch=$(mktemp -d)
|
||||
trap 'set +e; rm -rf "$scratch"' EXIT
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
# ~300 chars: over the old 236 ceiling, well under any POSIX PATH_MAX. The
|
||||
|
||||
@@ -23,8 +23,7 @@ cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
|
||||
@@ -32,8 +32,7 @@ cleanup() {
|
||||
stop
|
||||
rm -f "${log}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
|
||||
@@ -26,8 +26,7 @@ cleanup() {
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
rm -f "${log}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
|
||||
@@ -30,7 +30,7 @@ cleanup() {
|
||||
stop_server "$ptpid"
|
||||
rm -rf "$dir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup EXIT
|
||||
|
||||
# Neither header is present, so file->contenttype and ->lastmodified stay "".
|
||||
printf 'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
|
||||
|
||||
@@ -17,8 +17,7 @@ command -v httrack >/dev/null || {
|
||||
}
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_symbolize.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "$tmpdir"' EXIT
|
||||
trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
|
||||
trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
|
||||
out="${tmpdir}/trace"
|
||||
raw="${tmpdir}/trace-optout"
|
||||
overflow="this string is far too long for the buffer"
|
||||
|
||||
@@ -32,8 +32,7 @@ cleanup() {
|
||||
srv=
|
||||
rm -rf "${work}" "${srvlog}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
|
||||
# ~/.httrack.ini out of it.
|
||||
|
||||
@@ -38,8 +38,7 @@ cleanup() {
|
||||
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
rm -rf "${work}" "${srvlog}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# Stand in for a finished mirror: the crawl itself is not under test.
|
||||
proj="${work}/websites/proj"
|
||||
|
||||
@@ -27,8 +27,7 @@ cleanup() {
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
rm -f "${log}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
|
||||
@@ -34,8 +34,7 @@ cleanup() {
|
||||
wait "${csrv}" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
rm -rf "${log}" "${work}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
|
||||
@@ -28,8 +28,7 @@ cleanup() {
|
||||
rm -f "${log}"
|
||||
rm -rf "${base}"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
freeport() {
|
||||
python3 -c 'import socket
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user