mirror of
https://github.com/xroche/httrack.git
synced 2026-07-12 11:57:30 +03:00
Compare commits
2 Commits
ci-paralle
...
fix-test19
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee550c3972 | ||
|
|
145ea4769f |
32
.github/workflows/ci.yml
vendored
32
.github/workflows/ci.yml
vendored
@@ -55,9 +55,7 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -101,8 +99,7 @@ jobs:
|
||||
sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
|
||||
-exec mv {} {}.hidden \;
|
||||
! command -v python3
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -136,15 +133,7 @@ jobs:
|
||||
run: make -j"$(sysctl -n hw.ncpu)"
|
||||
|
||||
- name: Test
|
||||
# bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
|
||||
# it competes with other crawls, flaking its exact file count (the #527
|
||||
# macOS drop). Run everything else in parallel, then bigcrawl alone (its
|
||||
# serial-safe condition). Linux tolerates the full parallel run.
|
||||
run: |
|
||||
jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
|
||||
make check -j"$jobs" TESTS="$rest"
|
||||
make check TESTS=36_local-bigcrawl.test
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -182,9 +171,7 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -239,9 +226,7 @@ jobs:
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -287,8 +272,7 @@ jobs:
|
||||
# 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
|
||||
# skipped here (uninstrumented libz floods MSan with false positives).
|
||||
tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs" TESTS="$tests"
|
||||
make check TESTS="$tests"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
@@ -363,9 +347,7 @@ jobs:
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
|
||||
make check -j"$jobs"
|
||||
run: make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
|
||||
15
AGENTS.md
15
AGENTS.md
@@ -6,19 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
## Build & test
|
||||
- Fresh clone first: `git submodule update --init src/coucal`
|
||||
- `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
|
||||
automake, libtool), then `bash configure && make -j"$(nproc)" && make check
|
||||
-j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
|
||||
automake's parallel harness and each crawl test binds its own ephemeral-port
|
||||
server, so `-j` never contends and a multi-minute serial run drops to
|
||||
seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
|
||||
automatically; only a test slower than the current longest raises the floor.
|
||||
On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
|
||||
spend much of their wall time asleep (server trickles, httrack self-pacing),
|
||||
so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
|
||||
36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
|
||||
the macOS loopback when it competes with other crawls and flakes its exact
|
||||
file count (Linux tolerates the full parallel run).
|
||||
Or run `sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
automake, libtool), then `bash configure && make && make check`. Or run
|
||||
`sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
|
||||
## Hard invariants
|
||||
- **Generated autotools files are NOT in git.** `configure`, every
|
||||
|
||||
@@ -66,9 +66,12 @@ test -n "$port" || {
|
||||
}
|
||||
|
||||
out="$tmpdir/crawl"
|
||||
# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
|
||||
# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
|
||||
# bound the stall without changing what the fallback exercises.
|
||||
HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
|
||||
httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
|
||||
-c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
|
||||
-c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
|
||||
|
||||
log="$out/hts-log.txt"
|
||||
|
||||
@@ -92,12 +95,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# every address refused: the slot exhausts the list, then errors out (the
|
||||
# harness timeout would catch a hang/loop; refused connects are instant)
|
||||
# every address dead: the slot exhausts the list, then errors out (the harness
|
||||
# timeout would catch a hang/loop)
|
||||
out2="$tmpdir/crawl2"
|
||||
HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
|
||||
httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
|
||||
-c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
|
||||
-c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
|
||||
log2="$out2/hts-log.txt"
|
||||
|
||||
grep -q "trying next address" "$log2" || {
|
||||
|
||||
@@ -17,7 +17,10 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
|
||||
printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
|
||||
>"$tmproot/javaclass/Foo.class"
|
||||
|
||||
# --log-found first: a plugin-load failure (wrong soname, shadowing system
|
||||
# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
|
||||
--log-found '\+libhtsjava' \
|
||||
--found 'javaclass/Foo.class' \
|
||||
--found 'javaclass/hello.gif' \
|
||||
httrack 'BASEURL/javaclass/index.html'
|
||||
|
||||
Reference in New Issue
Block a user