Compare commits
33 Commits
docs-squas
...
cli-guide-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d5c16b7fe | ||
|
|
49ab161dcb | ||
|
|
1d0e2222d4 | ||
|
|
4166200465 | ||
|
|
2382ca3aa0 | ||
|
|
68a9a247d6 | ||
|
|
86bc02d8da | ||
|
|
e675d60301 | ||
|
|
1c8b93d9e7 | ||
|
|
d267aaf1bf | ||
|
|
226702fcab | ||
|
|
491774abda | ||
|
|
dbad05fcf0 | ||
|
|
db19089448 | ||
|
|
eb9e353f0b | ||
|
|
8a8fd7f9c0 | ||
|
|
1c1c0b1267 | ||
|
|
fe685d4117 | ||
|
|
f12d5e7434 | ||
|
|
e01e1b34de | ||
|
|
a67eb57def | ||
|
|
6634d48c21 | ||
|
|
a2c75b5316 | ||
|
|
6aabb3ba09 | ||
|
|
088f0711b8 | ||
|
|
ce481efeca | ||
|
|
7b68d7dc96 | ||
|
|
a47b7bee4d | ||
|
|
7be61d3111 | ||
|
|
3cc49b79c3 | ||
|
|
959deb0afb | ||
|
|
18bdc24d15 | ||
|
|
0b772ec6ba |
8
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Bumps builtin-baseline in src/vcpkg.json so the pinned OpenSSL/zlib can't rot.
|
||||
# windows-build.yml validates each bump; the pin makes what ships reproducible.
|
||||
- package-ecosystem: vcpkg
|
||||
directory: /src
|
||||
schedule:
|
||||
interval: weekly
|
||||
28
.github/workflows/windows-build.yml
vendored
@@ -44,6 +44,20 @@ jobs:
|
||||
shell: pwsh
|
||||
run: vcpkg integrate install
|
||||
|
||||
# The runner image's vcpkg checkout is pinned to some commit; our manifest's
|
||||
# builtin-baseline is usually newer, so `git show <baseline>:versions/...`
|
||||
# fails until that commit is local. Fetch exactly the pinned baseline (read
|
||||
# from the manifest, so Dependabot bumps need no workflow edit).
|
||||
- name: Fetch the pinned vcpkg baseline
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
$root = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { "C:\vcpkg" }
|
||||
$baseline = (Get-Content src/vcpkg.json -Raw | ConvertFrom-Json).'builtin-baseline'
|
||||
if (-not $baseline) { throw "no builtin-baseline in src/vcpkg.json" }
|
||||
git -C $root fetch --no-tags origin $baseline
|
||||
if ($LASTEXITCODE -ne 0) { throw "could not fetch vcpkg baseline $baseline" }
|
||||
|
||||
# httrack and webhttrack carry a ProjectReference to libhttrack, so building
|
||||
# them builds it first; proxytrack is standalone.
|
||||
- name: Build
|
||||
@@ -158,13 +172,21 @@ jobs:
|
||||
|
||||
pass=0 fail=0 skip=0 failed="" skipped=""
|
||||
for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
|
||||
*_local-*.test 13_crawl_proxy_https.test 58_watchdog.test; do
|
||||
*_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
|
||||
60_crawl-log-salvage.test; do
|
||||
rc=0
|
||||
run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
case "$rc" in
|
||||
0) pass=$((pass + 1)); echo "PASS $t" ;;
|
||||
77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
|
||||
124) fail=$((fail + 1)) failed="$failed $t"; echo "FAIL $t (timed out, tree killed)" ;;
|
||||
124)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (timed out, tree killed)"
|
||||
# Re-running a wedge traced would just hang again: salvage the
|
||||
# killed crawl's own logs into the artifact instead.
|
||||
dump_crawl_logs >>"$t.log"
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
*)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (exit $rc)"
|
||||
@@ -180,7 +202,7 @@ jobs:
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
expected_skips=" 48_local-crange-memresume.test" # pending #581
|
||||
expected_skips=" 48_local-crange-memresume.test 71_local-crange-repaircache.test" # pending #581
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
3
.gitignore
vendored
@@ -43,3 +43,6 @@ Makefile
|
||||
|
||||
# Python bytecode (tests/local-server.py).
|
||||
__pycache__/
|
||||
|
||||
# Per-checkout Claude Code rules (symlink into a local sandbox).
|
||||
/CLAUDE.local.md
|
||||
|
||||
30
AGENTS.md
@@ -19,6 +19,13 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
(`request_queue_size`) so macOS/BSD don't drop connections under a parallel
|
||||
`-c16` bigcrawl the way Python's default backlog of 5 did.
|
||||
Or run `sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
- A `tests/NN_*.test` runs only if listed in `tests/Makefile.am`'s `TESTS`; an
|
||||
unregistered file is silently skipped.
|
||||
- `make check` prepends the build's `src/` to `PATH`, but a hand-run `.test` does
|
||||
not — an installed `/usr/bin/httrack` then shadows your build. Run via `make
|
||||
check`, or `PATH="<bld>/src:$PATH"` for a manual run.
|
||||
- Give new `.test` scripts `set -e`: the older ones predate the rule, so several
|
||||
`local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
|
||||
|
||||
## Hard invariants
|
||||
- **Generated autotools files are NOT in git.** `configure`, every
|
||||
@@ -31,15 +38,32 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
- **Format only changed lines** with `git clang-format` (clang-format 19). Never
|
||||
reformat untouched code: the engine was formatted by an old tool and won't
|
||||
round-trip.
|
||||
- **Byte-safe edits.** Files with raw high bytes are ISO-8859-1 (French
|
||||
comments). Edit them byte-wise (`perl -0pi`, `sed`), not through a tool that
|
||||
re-encodes to UTF-8 and corrupts them.
|
||||
- **Byte-safe edits.** A few tracked files carry raw ISO-8859-1 high bytes
|
||||
(French comments): `src/htsconcat.c`, `lang/*.txt`, `html/contact.html`, and
|
||||
the `fuzz/corpus/*` vectors. Edit those byte-wise (`perl -0pi`, `sed`), not
|
||||
through a tool that re-encodes to UTF-8 and corrupts them. The rest of the tree
|
||||
is UTF-8 and safe to edit normally.
|
||||
|
||||
## Security (HTTrack parses hostile input off the network)
|
||||
- Bounds-check every copy. Overflow-safe form: put the untrusted value alone,
|
||||
`untrusted < limit - controlled` — never `controlled + untrusted < limit`,
|
||||
which can wrap and pass.
|
||||
|
||||
## C conventions
|
||||
- **Use the `*t` allocator wrappers, never raw libc** (`htssafe.h`):
|
||||
`malloct`/`calloct`/`realloct`/`freet`/`strdupt`, in test and selftest code
|
||||
too. `freet` NULLs its (lvalue) argument and tolerates NULL; `calloct(n, sz)`
|
||||
keeps calloc's arg order. Only exception: storing or calling a libc symbol
|
||||
itself (e.g. a resolver-backend function pointer).
|
||||
- **Exported API is `HTSEXT_API`.** Everything else is hidden by
|
||||
`-fvisibility=hidden` and free to change (check with `nm -D --defined-only
|
||||
libhttrack.so`). Touching an installed-header struct (see `DevIncludes_DATA` in
|
||||
`src/Makefile.am`) or an exported signature is an ABI break — flag and discuss,
|
||||
bump the soname, and prefer keeping the old entry point beside a new one.
|
||||
- **Windows ABI is free to break, POSIX is not.** The Windows DLL ships next to
|
||||
the exe with no soname contract, so a `_WIN32`-only ABI change needs no
|
||||
deprecation dance; POSIX/ELF keeps the flag-discuss-bump rules.
|
||||
|
||||
## Code & prose
|
||||
- Be terse. Comment the why, in English; translate French comments you touch.
|
||||
- Strip AI tells from prose (em-dash overuse, rule-of-three, filler, vague
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
/home/roche/git/httrack-works/CLAUDE.httrack.local.md
|
||||
12
configure.ac
@@ -1,6 +1,6 @@
|
||||
AC_PREREQ([2.71])
|
||||
|
||||
AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_INIT([httrack], [3.49.13], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_COPYRIGHT([
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2015 Xavier Roche and other contributors
|
||||
@@ -29,11 +29,13 @@ AC_CONFIG_SRCDIR(src/httrack.c)
|
||||
AC_CONFIG_MACRO_DIR([m4])
|
||||
AC_CONFIG_HEADERS(config.h)
|
||||
AM_INIT_AUTOMAKE([subdir-objects])
|
||||
# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
|
||||
# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
|
||||
# so it stays soname .so.3; bump revision.
|
||||
# 3:5:0: 3.49.13 leaves every installed struct layout and the exported symbol set
|
||||
# untouched vs 3.49.12; the 2 GB (_stat64) and UTF-8 export widenings are _WIN32-only,
|
||||
# and INTsys widened on POSIX (#600) but sits in no installed struct. Stays soname
|
||||
# .so.3; bump revision. x32 alone changes layout (LLint was 4 bytes there, #524), and
|
||||
# is not a release architecture.
|
||||
# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
|
||||
VERSION_INFO="3:4:0"
|
||||
VERSION_INFO="3:5:0"
|
||||
AM_MAINTAINER_MODE
|
||||
AC_USE_SYSTEM_EXTENSIONS
|
||||
|
||||
|
||||
15
debian/changelog
vendored
@@ -1,3 +1,18 @@
|
||||
httrack (3.49.13-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: SOCKS5 and CONNECT proxy support, brotli and zstd
|
||||
content decoding, --update data-loss fixes, and hardening of several
|
||||
network-facing parsers; full list in history.txt.
|
||||
* Add libbrotli-dev and libzstd-dev to Build-Depends for the new content
|
||||
codings.
|
||||
* libhttrack3 and libhttrack-dev no longer ship libhtsjava.so.3*: the
|
||||
obsolete Java-applet parser was removed upstream. It was a dlopen plugin
|
||||
with no reverse dependencies, so this is a plain file drop on upgrade;
|
||||
libhttrack.so.3 itself is unchanged and needs no rename.
|
||||
* Drop the htsjava attribution from debian/copyright.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Fri, 17 Jul 2026 10:49:52 +0200
|
||||
|
||||
httrack (3.49.12-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: security and crawl-correctness fixes (remote stack
|
||||
|
||||
25
history.txt
@@ -4,6 +4,31 @@ HTTrack Website Copier release history:
|
||||
|
||||
This file lists all changes and fixes that have been made for HTTrack
|
||||
|
||||
3.49-13
|
||||
+ New: SOCKS5 proxy support, with scheme-aware -P URLs (socks5://, socks5h://, connect://) and plain HTTP tunneled through a CONNECT-only proxy (#563, #564)
|
||||
+ New: decode brotli and zstd content codings, advertised over TLS only as browsers do (#556)
|
||||
+ New: webhttrack exposes the engine options added since 3.49-2, among them --cookies-file, --pause and --strip-query (#587)
|
||||
+ Fixed: files of 2 GB or more were mishandled on Windows and on every 32-bit build (#569)
|
||||
+ Fixed: --update destroyed a good local copy when the re-fetch returned an HTTP error, was aborted by -M/-E, failed to decode, or came in short (#176, #521, #557, #562)
|
||||
+ Fixed: a self-redirect cookie wall was dropped instead of being re-fetched with the cookie (#15)
|
||||
+ Fixed: a stalled TLS handshake ignored --timeout, and synchronous DNS resolution could wedge a crawl past --max-time (#607, #613)
|
||||
+ Fixed: -M metered saved bytes rather than received volume, and overshot its cap under a slow server (#77, #520)
|
||||
+ Fixed: several network-facing overflows and denial-of-service paths in the Content-Range, chunked-transfer, cookie, filter and ProxyTrack cache parsers
|
||||
+ Fixed: a failed connect did not fall back to the next address on Windows (#579)
|
||||
+ Fixed: -P took an out-of-range port as a garbage port, and scanned past an IPv6 literal's closing bracket (#598, #602)
|
||||
+ Fixed: reject a port outside 1..65535 wherever one is parsed (a crawled URL, the htsserver and proxytrack listen arguments, an ftp:// URL), instead of letting a bare sscanf wrap a huge value into a plausible port and silently use it (#614, #626)
|
||||
+ Fixed: a configured proxy still resolved and dialed the origin itself (#592)
|
||||
+ Fixed: ~/ in the -O base path was never expanded (#270)
|
||||
+ Fixed: a non-ASCII -O output path was double-encoded on Windows once argv became UTF-8 (#621)
|
||||
+ Fixed: files under a non-ASCII project path were saved to a mangled directory on Windows (#217)
|
||||
+ Fixed: --build-top-index (-%i) and --protocol (-@i) were taken for the -i continue flag, wiping the URL list and exiting on the usage screen (#615)
|
||||
+ Fixed: webhttrack ignored LC_ALL/LC_MESSAGES and picked the wrong Chinese and Portuguese (#95)
|
||||
+ Fixed: webhttrack wrote its base path and httrack.ini to the filesystem root when $HOME was empty (#625)
|
||||
+ Fixed: crawls on a non-default port were slowed by a per-request pre-resolve (#181)
|
||||
+ Changed: Windows builds moved to Visual Studio 2022 and OpenSSL 3.x, the VS2008 project files are retired, and the binaries carry a version resource
|
||||
+ Changed: removed the obsolete Java-applet (.class) parser and the dead SWF module remnants
|
||||
+ Changed: multiple internal hardening, test and CI improvements (Windows and macOS crawl suites, HTML-parser fuzzing, parallel make check)
|
||||
|
||||
3.49-12
|
||||
+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
|
||||
+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
|
||||
|
||||
@@ -579,7 +579,7 @@ And then, put the email address in your pages through:
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
223
html/android.html
Normal file
@@ -0,0 +1,223 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="How to mirror a website with HTTrack on Android: install the app, create a project, enter the address, run the mirror, and browse the result on your device." />
|
||||
<meta name="keywords" content="httrack, HTTrack, android, offline browser, web mirror utility, website mirroring, mobile, Google Play" />
|
||||
<title>HTTrack on Android</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>HTTrack on Android</em></h2>
|
||||
|
||||
<br>
|
||||
|
||||
<p>HTTrack downloads a website to your device so you can read it offline. The
|
||||
Android app runs the same mirroring engine as the desktop version behind a touch
|
||||
interface. It needs Android 7.0 or later, and is available on
|
||||
<a href="https://play.google.com/store/apps/details?id=com.httrack.android">Google Play</a>.</p>
|
||||
|
||||
<p>The steps below follow one mirror from start to finish.</p>
|
||||
|
||||
<br>
|
||||
|
||||
<ol>
|
||||
|
||||
<li><b>Grant storage access</b></li>
|
||||
<br><small>On first launch the app asks for permission to store mirrors on your
|
||||
device. The prompt reads <em>"Allow HTTrack Website Copier to access photos,
|
||||
media, and files on your device?"</em>. Tap <b>ALLOW</b>: without it the app
|
||||
cannot save the downloaded files.</small>
|
||||
<br><br><center><img src="img/android_permission.png" width="320" alt="First-run storage permission dialog" border="0"></center>
|
||||
<br><small>If you have used an older release, a second prompt offers to bring its
|
||||
mirrors into the app. Tap <b>Import</b> to move them, or <b>Not now</b> to
|
||||
skip.</small>
|
||||
<br><br><center><img src="img/android_import.png" width="320" alt="Import mirrors from an older version" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Start</b></li>
|
||||
<br><small>The welcome screen shows the engine version at the bottom. Tap
|
||||
<b>Next</b> to create a project, or <b>Browse sites</b> to open a mirror you
|
||||
already made.</small>
|
||||
<br><br><center><img src="img/android_startup.png" width="320" alt="Welcome screen" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Name the project</b></li>
|
||||
<br><small>Give the project a name, and optionally a category to group related
|
||||
mirrors. <b>Base path</b> shows where the files will be written. Tap
|
||||
<b>Next</b>.</small>
|
||||
<br><br><center><img src="img/android_project.png" width="320" alt="Project name, category, and base path" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Enter the address</b></li>
|
||||
<br><small>Type the site address. If a project of the same name already exists,
|
||||
pick <b>Continue interrupted download</b> or <b>Update existing download</b>.
|
||||
Tap <b>Options</b> to adjust the crawl, or <b>Start</b> to begin.</small>
|
||||
<br><br><center><img src="img/android_url.png" width="320" alt="Web address and download action" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Options (optional)</b></li>
|
||||
<br><small>The <b>Options</b> screen holds the crawl settings, grouped into
|
||||
eleven tabs. It uses the same profile format as the desktop version, so a tab you
|
||||
know from WinHTTrack behaves the same way here.</small>
|
||||
<br><br><center><img src="img/android_options.png" width="320" alt="Options tab list" border="0"></center>
|
||||
<br><small>The tabs are:</small>
|
||||
<ul>
|
||||
<li><small><b>Scan Rules</b>: wildcard filters for the addresses to keep or skip.</small></li>
|
||||
<li><small><b>Limits</b>: caps on depth, size, time, speed, and number of connections.</small></li>
|
||||
<li><small><b>Flow Control</b>: simultaneous connections, timeouts, and retries.</small></li>
|
||||
<li><small><b>Links</b>: how far to follow links, and which extra files to fetch.</small></li>
|
||||
<li><small><b>Build</b>: how the saved files and folders are named.</small></li>
|
||||
<li><small><b>Browser ID</b>: the user-agent, language, and headers sent to servers.</small></li>
|
||||
<li><small><b>Spider</b>: cookies, robots.txt handling, and request behavior.</small></li>
|
||||
<li><small><b>Proxy address</b>: proxy host and port.</small></li>
|
||||
<li><small><b>Log, Index, Cache</b>: log files, the search index, and the update cache.</small></li>
|
||||
<li><small><b>Type/MIME associations</b>: map file extensions to MIME types.</small></li>
|
||||
<li><small><b>Experts Only</b>: scan mode, how far the crawl may travel, and link rewriting.</small></li>
|
||||
</ul>
|
||||
<br><center><img src="img/android_scanrules.png" width="320" alt="Scan Rules tab" border="0"></center>
|
||||
<br><center><img src="img/android_experts.png" width="320" alt="Experts Only tab" border="0"></center>
|
||||
<br><small>The <a href="step9.html">desktop option reference</a> describes every
|
||||
setting in full. A few desktop options are missing or fixed on Android: the proxy
|
||||
tab has a host and port but no login and password, the download folder is fixed
|
||||
inside the app's storage so there is no output-path setting, and the MIME table
|
||||
holds up to eight entries.</small>
|
||||
<br><br>
|
||||
|
||||
<li><b>Run the mirror</b></li>
|
||||
<br><small>HTTrack downloads the site and reports live figures: bytes saved,
|
||||
links scanned, transfer rate, and errors. Tap <b>Abort</b> to stop early; a
|
||||
partial mirror can be resumed later.</small>
|
||||
<br><br><center><img src="img/android_progress.png" width="320" alt="Crawl in progress with live statistics" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Browse the result</b></li>
|
||||
<br><small>When the crawl finishes the app reports <b>Success</b> and the mirror
|
||||
location. Tap <b>Browse Mirrored Website</b> to read the copy in your browser,
|
||||
<b>View log</b> to see what happened, or <b>New project</b> to start
|
||||
again.</small>
|
||||
<br><br><center><img src="img/android_finished.png" width="320" alt="Mirror finished, success" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Where the files are</b></li>
|
||||
<br><small>Mirrors are written under <tt>/storage/emulated/0/HTTrack/Websites</tt>,
|
||||
in a folder named after the project. That folder lives in your device's shared
|
||||
storage, so a file manager or a USB connection can reach it. Opening
|
||||
<tt>index.html</tt> there browses a mirror without the app.</small>
|
||||
|
||||
</ol>
|
||||
|
||||
<br><br><br><br>
|
||||
<p align="right">Back to <a href="index.html">Home</a></p>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -173,7 +173,7 @@ There are also specific issues regarding this format:
|
||||
<ul>
|
||||
<li>The data in the central directory (such as CD extra field, and CD comments) are not used</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^16 files (65535) ; in such case the total number of entries in the 32-bit central directory is 65536 (0xffff), but the presence of the 64-bit central directory is not mandatory</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory must be present <b>(not currently supported)</b></li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory is emitted automatically (a single stored entry of 4GiB or more is not supported)</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
@@ -282,7 +282,7 @@ Libraries should generally handle this peculiar format, however.
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -106,6 +106,8 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
The command-line version
|
||||
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
<br>A task-oriented guide: how to do the common things, and the defaults that surprise newcomers<br><br>
|
||||
<li><a href="options.html">Command line Options</a></li>
|
||||
<br>List of all powerful command line options<br><br>
|
||||
<li>How to use httrack command-line version:</li>
|
||||
@@ -145,7 +147,7 @@ The command-line version
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
496
html/cmdguide.html
Normal file
@@ -0,0 +1,496 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>Command-Line Guide</em></h2>
|
||||
|
||||
<p>This is a task-oriented guide to the <tt>httrack</tt> command line: how to do the
|
||||
things people actually ask for, and the handful of defaults that surprise
|
||||
newcomers. It sits on top of the
|
||||
<a href="httrack.man.html">generated manual page</a>, which lists every option in
|
||||
full. When you want the exhaustive detail for a flag, that page is the reference;
|
||||
this one is the map.</p>
|
||||
|
||||
<p>Two habits before anything else. First, HTTrack has its own options: they are not
|
||||
wget or curl flags, so reach for the tables here rather than guessing. Second,
|
||||
when a mirror does something you did not expect, the answer is almost always in
|
||||
the log. Every project writes <tt>hts-log.txt</tt> (and <tt>hts-err.txt</tt>) into
|
||||
its output directory, and those files name every URL that was refused, redirected,
|
||||
or filtered out. Read them first.</p>
|
||||
|
||||
<h4>On this page</h4>
|
||||
<ul class="tblNoBorder">
|
||||
<li><a href="#quickstart">1. Quick start</a></li>
|
||||
<li><a href="#scope">2. Scope: how far the crawl reaches</a></li>
|
||||
<li><a href="#filters">3. Filters and scan rules</a></li>
|
||||
<li><a href="#limits">4. Limits and politeness</a></li>
|
||||
<li><a href="#names">5. File names and types</a></li>
|
||||
<li><a href="#links">6. Links and page building</a></li>
|
||||
<li><a href="#identity">7. Identity, cookies and login</a></li>
|
||||
<li><a href="#proxy">8. Proxy and network</a></li>
|
||||
<li><a href="#update">9. Update and cache</a></li>
|
||||
<li><a href="#experts">10. Experts and scripting</a></li>
|
||||
<li><a href="#recipes">11. Recipes</a></li>
|
||||
</ul>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="quickstart">1. Quick start</h3>
|
||||
|
||||
<p>A mirror is one command: a start URL and an output directory.</p>
|
||||
|
||||
<p><tt>httrack https://example.com/ -O mydir</tt></p>
|
||||
|
||||
<p>With no other options HTTrack mirrors that site, stays on the same host, follows
|
||||
links to any depth, rebuilds them to browse offline, and stores everything under
|
||||
<tt>mydir</tt>. The same directory also holds the log files and the
|
||||
<tt>hts-cache/</tt> folder that makes a later update or resume possible.</p>
|
||||
|
||||
<p>Two defaults are worth knowing up front, because both catch people out:</p>
|
||||
<ul>
|
||||
<li>HTTrack throttles itself to about <b>100 KB/s</b> even when you pass no rate
|
||||
option. If a mirror feels slow, that is why. See
|
||||
<a href="#limits">Limits</a> for how to lift it.</li>
|
||||
<li>The download proceeds as a well-behaved robot: it identifies itself as
|
||||
<tt>HTTrack</tt>, obeys <tt>robots.txt</tt>, and sends a Referer with each
|
||||
request. A site that blocks that behavior needs the levers in
|
||||
<a href="#identity">Identity</a>, not brute force.</li>
|
||||
</ul>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="scope">2. Scope: how far the crawl reaches</h3>
|
||||
|
||||
<p>Scope decides which links HTTrack is even willing to follow, before any filter
|
||||
you write. Get this right and most "it downloaded too much" or "it only grabbed
|
||||
the index" problems disappear.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>-rN</tt></td><td>Maximum link depth. <b>The start page is level 1</b>, so one level of links out is <tt>-r2</tt>, not <tt>-r1</tt>.</td></tr>
|
||||
<tr><td><tt>-a -d -l -e</tt></td><td>How far off the starting host the crawl may travel: same address (host), same principal domain, same top-level domain (for example .com), or everywhere. The default keeps you on the starting host.</td></tr>
|
||||
<tr><td><tt>-D -U -S -B</tt></td><td>Directory travel: down into subdirectories only, up to parent directories only, stay in the same directory, or both up and down.</td></tr>
|
||||
<tr><td><tt>-n</tt></td><td>Also fetch non-HTML files "near" a followed link, such as an image linked from a page you kept but hosted elsewhere.</td></tr>
|
||||
<tr><td><tt>-%eN</tt></td><td>How many levels of external links to follow once the crawl leaves your scope (default 0).</td></tr>
|
||||
<tr><td><tt>-t</tt></td><td>Also HEAD-test links that fall outside the scope, which are normally refused, without downloading them: a way to see what scope is excluding.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>The single most common surprise is "only the home page came down." That is
|
||||
usually not a scope option at all: it is an off-host redirect. A start URL of
|
||||
<tt>http://example.com/</tt> that redirects to <tt>https://www.example.com/</tt>
|
||||
lands you on a different host, and same-host scope stops the crawl there. Start
|
||||
from the final URL, or add a filter that re-admits the real host (see
|
||||
<a href="#filters">Filters</a>). The log will show the redirect.</p>
|
||||
|
||||
<p><tt>-n</tt> is the fix for pages that render locally without their images or
|
||||
stylesheets: it lets HTTrack pull in requisites that sit just outside scope. Note
|
||||
that its embedded-asset handling (following <tt>img</tt>, <tt>link</tt>,
|
||||
<tt>script</tt>, <tt>style</tt> and HTML5 <tt>source</tt>/<tt>track</tt> targets
|
||||
past the normal depth and filter limits) applies only when <tt>-n</tt> is on; it
|
||||
is not automatic. It can also over-fetch by dragging in a whole external host from
|
||||
a single link, in which case name the assets you want with a filter instead.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="filters">3. Filters and scan rules</h3>
|
||||
|
||||
<p>Filters are the number-one source of confusion, and also the tool that solves
|
||||
most scope problems once you understand them. A filter is a rule that accepts
|
||||
(<tt>+</tt>) or rejects (<tt>-</tt>) URLs by pattern. The sign is mandatory:
|
||||
<tt>+pattern</tt> adds, <tt>-pattern</tt> removes, and a bare pattern is an error.</p>
|
||||
|
||||
<p>The rules that matter:</p>
|
||||
<ul>
|
||||
<li><b>Last match wins.</b> Rules are applied in order and the last one that
|
||||
matches a URL decides its fate. Order your rules from general to specific.</li>
|
||||
<li><b>Wildcards.</b> <tt>*</tt> matches any run of characters;
|
||||
<tt>*[a-z]</tt>, <tt>*[0-9]</tt> and similar classes match sets. So
|
||||
<tt>+*.pdf</tt> means "any URL ending in .pdf".</li>
|
||||
<li><b>Whitelisting.</b> To keep one site and nothing else, deny everything then
|
||||
re-admit the host: <tt>"-*" "+example.com/*"</tt>. A lone <tt>+</tt> rule only
|
||||
adds to the default scope; it never restricts.</li>
|
||||
<li><b>Size rules.</b> <tt>*[>100000]</tt> and <tt>*[<1000]</tt> filter by
|
||||
byte size. Because size is only known once the transfer starts, an oversize file
|
||||
is fetched partway and then aborted, not skipped for free.</li>
|
||||
<li><b>mime: rules.</b> A rule like <tt>-mime:video/*</tt> matches the
|
||||
<tt>Content-Type</tt>. That type is only known after the response headers arrive,
|
||||
so a mime rule <b>cannot stop a request</b>; it can only abort the body. Use a
|
||||
URL pattern when you want to avoid the fetch entirely.</li>
|
||||
</ul>
|
||||
|
||||
<p>Quote your filters. Shells treat <tt>*</tt>, <tt>[</tt> and sometimes <tt>+</tt>
|
||||
specially, so wrap each rule in quotes as shown above. The full pattern language,
|
||||
with tables for wildcards, size and mime, is in
|
||||
<a href="filters.html">the filters page</a>, and the
|
||||
<a href="faq.html">FAQ</a> has a worked tutorial.</p>
|
||||
|
||||
<p><b>robots.txt.</b> By default HTTrack obeys <tt>robots.txt</tt> (<tt>-s2</tt>).
|
||||
<tt>-s0</tt> ignores it entirely, <tt>-s1</tt> obeys it but lets one of your
|
||||
<tt>+</tt> filters override a disallow for a URL you explicitly asked for. Note
|
||||
that a <tt>403 Forbidden</tt> is a server refusal, not a robots rule: robots
|
||||
options will not help there. That is an
|
||||
<a href="#identity">identity</a> problem.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="limits">4. Limits and politeness</h3>
|
||||
|
||||
<p>HTTrack ships cautious on purpose: it is easy to hammer a small site by
|
||||
accident, and the <a href="abuse.html">abuse page</a> is worth a read. The limits
|
||||
below let you go faster when you own the target, and slower when you do not.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>-AN</tt></td><td>Maximum transfer rate in bytes/sec. <b>The default is about 100 KB/s even without this flag.</b> Raise it to go faster.</td></tr>
|
||||
<tr><td><tt>-cN</tt></td><td>Number of parallel connections (default 4). <tt>--tiny</tt>, <tt>--wide</tt> and <tt>--ultrawide</tt> are presets.</td></tr>
|
||||
<tr><td><tt>-%cN</tt></td><td>New connections opened per second (default 5).</td></tr>
|
||||
<tr><td><tt>-MN</tt></td><td>Stop after N bytes <b>received from the network</b> across the whole mirror (this counts what was transferred, not what was saved).</td></tr>
|
||||
<tr><td><tt>-EN</tt></td><td>Stop after N seconds of wall-clock time.</td></tr>
|
||||
<tr><td><tt>-mN</tt></td><td>Per-file size caps.</td></tr>
|
||||
<tr><td><tt>-TN -RN -JN -HN</tt></td><td>Idle timeout, retry count, minimum acceptable rate, and host-ban behavior for slow or dead hosts.</td></tr>
|
||||
<tr><td><tt>-GN -%GN</tt></td><td>Pause the mirror at N bytes, or pause between files, to spread the load.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>The security clamps.</b> To keep an accidental typo from turning into a flood,
|
||||
HTTrack silently caps a few values: at most 8 connections (<tt>-c</tt>), at most
|
||||
10 MB/s (<tt>-A</tt>), and at most 5 new connections per second (<tt>-%c</tt>).
|
||||
Ask for more and you get the ceiling, quietly. The single flag <tt>-%!</tt> lifts
|
||||
all three. Use it only against infrastructure you are allowed to load that hard.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="names">5. File names and types</h3>
|
||||
|
||||
<p>Where local files land, and what they are called, is controlled by the naming
|
||||
options. This is the second-biggest source of "why did it do that" questions,
|
||||
usually about a URL like <tt>/article?id=42</tt> or a <tt>.php</tt> page that is
|
||||
really HTML.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>-NN</tt></td><td>The local path and name layout. Presets are numeric, and you can also give a template such as <tt>-N "%h%p/%n%q.%t"</tt>.</td></tr>
|
||||
<tr><td><tt>-LN</tt></td><td>Long names, 8.3 names, or ISO9660 for CD masters.</td></tr>
|
||||
<tr><td><tt>-%A ext=type</tt></td><td>Assume a MIME type for an extension, for example <tt>-%A php=text/html</tt>. This also skips the extra HEAD probe HTTrack would otherwise send to learn the type.</td></tr>
|
||||
<tr><td><tt>-%N -%D -u -%t</tt></td><td>When and how the content type is checked, and whether the original extension is kept.</td></tr>
|
||||
<tr><td><tt>-%q -%g</tt></td><td>Whether the query string appears in the local filename, and whether query keys are stripped when deciding if two URLs are the same file.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>The <tt>-N</tt> presets are built from modular arithmetic on the name fields, so
|
||||
undocumented number combinations often "work" by accident. If you care about the
|
||||
exact layout, use an explicit template (the <tt>%h %p %n %q %t</tt> placeholders)
|
||||
rather than a magic number, and check the result on a small crawl first.</p>
|
||||
|
||||
<p>A dynamic page served as <tt>.php</tt> or <tt>.asp</tt> that is actually HTML is
|
||||
the classic case: without help it can be saved with an extension a browser will
|
||||
not open locally. <tt>-%A php=text/html</tt> fixes both the extension and the
|
||||
naming.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="links">6. Links and page building</h3>
|
||||
|
||||
<p>After a page is downloaded, HTTrack parses it for more links and rewrites the
|
||||
ones it kept so the local copy browses offline. These options tune both halves.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>-KN</tt></td><td>How links are rewritten in saved pages. The numbering is inverted from what you might guess: bare <tt>-K</tt> keeps <b>absolute</b> URLs, and <tt>-K0</tt> is the <b>relative</b> default. <tt>-K3</tt> keeps absolute URIs, <tt>-K4</tt> keeps the original links.</td></tr>
|
||||
<tr><td><tt>-x -o</tt></td><td>Replace external links with an error page, and generate an error page for links that failed.</td></tr>
|
||||
<tr><td><tt>-%p -%x</tt></td><td>Leave HTML untouched (no rewriting), and strip passwords out of saved links.</td></tr>
|
||||
<tr><td><tt>-%P -j</tt></td><td>Aggressive link discovery, and how much script content is parsed for links.</td></tr>
|
||||
<tr><td><tt>-%M</tt></td><td>Save the whole mirror as a single MIME-encapsulated <tt>.mht</tt> archive (<tt>index.mht</tt>).</td></tr>
|
||||
<tr><td><tt>-I -%i -%I</tt></td><td>Build a per-mirror index, a top index across projects, and a searchable keyword index.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>HTTrack finds links by parsing HTML and CSS. It does not run JavaScript, so any
|
||||
URL a page builds at runtime in script (a lazy-loaded image, a
|
||||
JavaScript-assembled path) is invisible to the crawler and will be missing from
|
||||
the mirror. There is no flag that fixes this; the asset has to appear in the
|
||||
static HTML or CSS to be found. <tt>-%P</tt> widens discovery for links that are
|
||||
present but awkwardly formatted, not for links that do not exist until script
|
||||
runs.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="identity">7. Identity, cookies and login</h3>
|
||||
|
||||
<p>By default HTTrack is an honest robot: it sends a <tt>User-Agent</tt> of
|
||||
<tt>HTTrack</tt>, a Referer with each link (which reveals the crawl path to the
|
||||
server), and obeys robots. Plenty of sites filter exactly that profile. These
|
||||
options control what HTTrack says about itself.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>-F "string"</tt></td><td>The <tt>User-Agent</tt>. Set a browser string to get past crawler blocks; <tt>-F ""</tt> sends none.</td></tr>
|
||||
<tr><td><tt>-%R -%E -%l -%a</tt></td><td>Referer, From, Accept-Language and Accept headers.</td></tr>
|
||||
<tr><td><tt>-%X "line"</tt></td><td>Add raw header lines to every request.</td></tr>
|
||||
<tr><td><tt>-%F</tt></td><td>A footer written into saved pages (on disk, not a network header).</td></tr>
|
||||
<tr><td><tt>-b -%K file</tt></td><td>Accept cookies, and preload a Netscape <tt>cookies.txt</tt>.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>Login.</b> For HTTP Basic auth, put the credentials in the URL:
|
||||
<tt>http://user:pass@host/</tt>. An <tt>@</tt> inside the username must be written
|
||||
<tt>%40</tt>. Only Basic is supported, not Digest.</p>
|
||||
|
||||
<p>For cookie or form logins, the simplest path is to log in with a browser, export
|
||||
its <tt>cookies.txt</tt>, and drop that file in the project directory so HTTrack
|
||||
sends the session cookie. For a form that needs a POST, <tt>--catchurl</tt> can
|
||||
capture the exact request your browser sends and replay it. A few cookie caveats
|
||||
to know: expiry is ignored, there is a silent cap of about 8 cookies sent per
|
||||
request, and <tt>-b0</tt> disables cookies and the reuse of Basic credentials
|
||||
across links at the same time.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="proxy">8. Proxy and network</h3>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>-P proxy:port</tt></td><td>Route through a proxy. HTTP, SOCKS5 and CONNECT are supported: <tt>-P host:8080</tt>, <tt>-P socks5://host:1080</tt>, <tt>-P connect://host:443</tt>, with optional <tt>user:pass@</tt>.</td></tr>
|
||||
<tr><td><tt>-%f</tt></td><td>Send FTP requests through the HTTP proxy.</td></tr>
|
||||
<tr><td><tt>-@iN</tt></td><td>Prefer IPv4 or IPv6.</td></tr>
|
||||
<tr><td><tt>-%h -%k -%z</tt></td><td>Force HTTP/1.0 (drops keep-alive and compression, useful for fragile CGI), toggle keep-alive, and toggle compression.</td></tr>
|
||||
<tr><td><tt>-%b -%B</tt></td><td>Bind to a local address, and accept technically-bogus responses some servers send.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>Two network facts worth stating plainly. Over SOCKS5, HTTrack always resolves
|
||||
host names at the proxy (remote DNS) for both <tt>socks5://</tt> and
|
||||
<tt>socks5h://</tt>, so your local resolver is never consulted. And HTTrack does
|
||||
not verify TLS certificates: HTTPS gives you an encrypted transport, but not an
|
||||
authenticated one. That is a deliberate choice for a mirroring tool, not a bug,
|
||||
but it is worth knowing if you are relying on it for trust.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="update">9. Update and cache</h3>
|
||||
|
||||
<p>Every project keeps a cache under <tt>hts-cache/</tt>. It records every URL that
|
||||
was fetched, together with the options you used, and it is what makes resuming and
|
||||
updating possible. It is not a size-limited scratch area you can delete: throw it
|
||||
away and you lose the ability to continue or update the mirror.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--continue</tt></td><td>Carry on an interrupted mirror, trusting the cache: it does not re-check pages already stored.</td></tr>
|
||||
<tr><td><tt>--update</tt></td><td>Re-run the mirror, revalidating each page with the server (If-Modified-Since / If-None-Match) and downloading only what changed.</td></tr>
|
||||
<tr><td><tt>-X0</tt></td><td>Do not purge. By default an update deletes local files that are no longer part of the mirror; <tt>-X0</tt> keeps them.</td></tr>
|
||||
<tr><td><tt>-C</tt></td><td>Cache mode. The default already does the right thing and switches to update-checking when it detects an existing mirror.</td></tr>
|
||||
<tr><td><tt>-%u -%j -%o -%y -%n -%s -k</tt></td><td>URL-deduplication behavior and cache storage details.</td></tr>
|
||||
<tr><td><tt>-#C -#R --clean</tt></td><td>Inspect the cache, repair its ZIP, and erase cache plus logs.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>The purge trap.</b> An <tt>--update</tt> run rebuilds the list of files the
|
||||
mirror should contain, then deletes any previously-mirrored file that is not on the
|
||||
new list. This is what keeps a mirror in sync with a shrinking site, but it means a
|
||||
partial or interrupted update can delete files you meant to keep. If an update might
|
||||
not complete cleanly, add <tt>-X0</tt> to protect the existing tree, and expect
|
||||
dynamic pages to look "changed" on every run. See the
|
||||
<a href="cache.html">cache page</a> for the details.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="experts">10. Experts and scripting</h3>
|
||||
|
||||
<p>HTTrack is also a scriptable fetch-and-scan tool. These options turn off the
|
||||
mirror behavior and expose the engine.</p>
|
||||
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Option</b></td><td><b>What it controls</b></td></tr>
|
||||
<tr><td><tt>--get URL</tt></td><td>Fetch a single file and stop. Cache, index, depth, cookies and robots are all off for this mode.</td></tr>
|
||||
<tr><td><tt>--spider --testlinks --skeleton</tt></td><td>Scan without saving, test links at depth 1, or keep HTML only. Handy for checking a site before a real crawl.</td></tr>
|
||||
<tr><td><tt>-V "cmd"</tt></td><td>Run a shell command on each downloaded file; <tt>$0</tt> is the file path. Good for on-the-fly processing.</td></tr>
|
||||
<tr><td><tt>-%W module</tt></td><td>Load an external callback module to hook the engine.</td></tr>
|
||||
<tr><td><tt>-Q -q -v -f -z -Z</tt></td><td>Logging: quiet, no questions, verbose on screen, and the various log-to-file levels.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>The <a href="dev.html">developer page</a> covers the callback API and batch use
|
||||
in more depth.</p>
|
||||
|
||||
<hr>
|
||||
|
||||
<h3 id="recipes">11. Recipes</h3>
|
||||
|
||||
<p>Copy-ready command lines for the tasks people ask about most. Each has the one
|
||||
gotcha that trips it up.</p>
|
||||
|
||||
<h4>Mirror one site, nothing off-host</h4>
|
||||
<p><tt>httrack https://example.com/ -O mydir</tt><br>
|
||||
<small>Same-host is already the default. The usual failure is a
|
||||
<tt>www.</tt>-to-apex or <tt>http</tt>-to-<tt>https</tt> redirect that moves you off
|
||||
host and stops the crawl; start from the final URL, or add
|
||||
<tt>"+finalhost/*"</tt>.</small></p>
|
||||
|
||||
<h4>One level of links only</h4>
|
||||
<p><tt>httrack https://example.com/ -r2 -O mydir</tt><br>
|
||||
<small>Depth counts the start page as level 1, so one level out is <tt>-r2</tt>.</small></p>
|
||||
|
||||
<h4>This site only, deny everything else</h4>
|
||||
<p><tt>httrack https://example.com/ "-*" "+example.com/*" -O mydir</tt><br>
|
||||
<small>You must deny-all first; a lone <tt>+</tt> only adds, and the last matching
|
||||
rule wins.</small></p>
|
||||
|
||||
<h4>Grab every PDF under a site</h4>
|
||||
<p><tt>httrack https://example.com/ "-*" "+example.com/*.pdf" -O mydir</tt><br>
|
||||
<small>Always scope the host. A bare <tt>+*.pdf</tt> invites the crawler onto the
|
||||
whole web.</small></p>
|
||||
|
||||
<h4>Keep page requisites, including off-host images</h4>
|
||||
<p><tt>httrack https://example.com/blog/ -n -O mydir</tt><br>
|
||||
<small><tt>-n</tt> can over-fetch by pulling in an entire external host from one
|
||||
link; if it does, name the assets with <tt>"+cdn.example.com/*"</tt> instead.</small></p>
|
||||
|
||||
<h4>Resume an interrupted crawl</h4>
|
||||
<p><tt>httrack --continue -O mydir</tt><br>
|
||||
<small>Needs an intact <tt>hts-cache/</tt>. Deleting it loses the URL list and your
|
||||
options.</small></p>
|
||||
|
||||
<h4>Update a mirror, downloading only what changed</h4>
|
||||
<p><tt>httrack --update -O mydir</tt><br>
|
||||
<small>Purge deletes any previously-mirrored file not seen this run; add
|
||||
<tt>-X0</tt> to protect the tree against a partial update.</small></p>
|
||||
|
||||
<h4>Reach a section behind a login cookie</h4>
|
||||
<p><small>Export your browser's <tt>cookies.txt</tt> into the project directory,
|
||||
then:</small><br>
|
||||
<tt>httrack https://example.com/members/ -O mydir</tt><br>
|
||||
<small>The file must be Netscape <tt>cookies.txt</tt> in the project folder;
|
||||
<tt>-b0</tt> would disable cookies and Basic-auth reuse together.</small></p>
|
||||
|
||||
<h4>Get past a crawler-UA block</h4>
|
||||
<p><tt>httrack https://example.com/ -F "Mozilla/5.0 (Windows NT 10.0)" -O mydir</tt><br>
|
||||
<small>A 403 is server-side, not robots: change the User-Agent, and add
|
||||
<tt>-%h</tt> for fragile CGI. Do not reach for <tt>-s0</tt>.</small></p>
|
||||
|
||||
<h4>Full-speed mirror on your own server</h4>
|
||||
<p><tt>httrack https://example.com/ -A2000000 -c8 -%! -O mydir</tt><br>
|
||||
<small>The default throttles to about 100 KB/s. <tt>-%!</tt> lifts the built-in
|
||||
caps; use it only against infrastructure you are allowed to load.</small></p>
|
||||
|
||||
<h4>HTTrack as a fetch tool</h4>
|
||||
<p><tt>httrack --get https://host/file.bin -O tmp</tt><br>
|
||||
<small><tt>--get</tt> fetches one file with cache, index, depth, cookies and robots
|
||||
all disabled.</small></p>
|
||||
|
||||
<p><br>For the complete, always-current option list, see
|
||||
<a href="httrack.man.html">the manual page</a>. For the filter language, see
|
||||
<a href="filters.html">filters</a>; for the cache and updates, see
|
||||
<a href="cache.html">cache</a>.</p>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
@@ -243,7 +243,7 @@ roche at httrack dot com (Xavier ROCHE)<br>
|
||||
<br><hr><br>
|
||||
<br>
|
||||
This program is covered by the GNU General Public License.<br>
|
||||
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2007 Xavier Roche and other contributors
|
||||
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2026 Xavier Roche and other contributors
|
||||
<br>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
@@ -259,7 +259,7 @@ roche at httrack dot com (Xavier ROCHE)<br>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -147,7 +147,7 @@ This page describes the HTTrack cache format.
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -263,7 +263,7 @@ clear language!
|
||||
|
||||
<li><a href="#QM8">Can HTTrack generates HP-UX or ISO9660 compatible files?</a><br></li>
|
||||
|
||||
<li><a href="#QM9">If there any SOCKS support?</a><br></li>
|
||||
<li><a href="#QM9">Is there any SOCKS support?</a><br></li>
|
||||
|
||||
<li><a href="#QM10">What's this hts-cache directory? Can I remove it?</a><br></li>
|
||||
|
||||
@@ -835,8 +835,8 @@ A: <em>Yes. Use user:password@your_proxy_name as your proxy name (example: <tt>s
|
||||
<br><br><a NAME="QM8">Q: <strong>Can HTTrack generates HP-UX or ISO9660 compatible files?</strong></a><br>
|
||||
A: <em>Yes. See the build options (-N, or see the WinHTTrack options)</em>
|
||||
|
||||
<br><br><a NAME="QM9">Q: <strong>If there any SOCKS support?</strong></a><br>
|
||||
A: <em>Not yet!</em>
|
||||
<br><br><a NAME="QM9">Q: <strong>Is there any SOCKS support?</strong></a><br>
|
||||
A: <em>Yes. HTTrack supports SOCKS5 and HTTP CONNECT proxies: give the proxy with a scheme prefix, e.g. <tt>-P socks5://host:port</tt> or <tt>-P connect://host:port</tt> (prefix <tt>user:pass@</tt> before the host for authenticated proxies).</em>
|
||||
|
||||
<br><br><a NAME="QM10">Q: <strong>What's this hts-cache directory? Can I remove it?</strong></a><br>
|
||||
A: <em>NO if you want to update the site, because this directory is used by HTTrack for this purpose.
|
||||
@@ -931,7 +931,7 @@ A: <em>Feel free to <a href="contact.html">contact us</a>!
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -189,7 +189,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<br>
|
||||
Filters are analyzed by HTTrack from the first filter to the last one. The complete URL
|
||||
name is compared to filters defined by the user or added automatically by HTTrack. <br><br>
|
||||
A scan rule has an higher priority is it is declared later - hierarchy is important: <br>
|
||||
A scan rule has a higher priority if it is declared later - hierarchy is important: <br>
|
||||
|
||||
<br>
|
||||
<table BORDER="1" CELLPADDING="2">
|
||||
@@ -307,7 +307,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<br>
|
||||
Filters are analyzed by HTTrack from the first filter to the last one. The sizes
|
||||
are compared against scan rules defined by the user.<br><br>
|
||||
A scan rule has an higher priority is it is declared later - hierarchy is important.<br>
|
||||
A scan rule has a higher priority if it is declared later - hierarchy is important.<br>
|
||||
|
||||
Note: scan rules based on size can be mixed with regular URL patterns<br>
|
||||
|
||||
@@ -361,7 +361,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<br>
|
||||
Filters are analyzed by HTTrack from the first filter to the last one. The complete MIME
|
||||
type is compared against scan rules defined by the user.<br><br>
|
||||
A scan rule has an higher priority is it is declared later - hierarchy is important<br>
|
||||
A scan rule has a higher priority if it is declared later - hierarchy is important<br>
|
||||
|
||||
Note: scan rules based on MIME types can <b>NOT</b> be mixed with regular URL patterns or size patterns within the same rule, but you can use both of them in distinct ones<br>
|
||||
|
||||
@@ -387,12 +387,12 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<tr>
|
||||
<td nowrap><tt>-mime:video/*</tt></td>
|
||||
<td>This will refuse all video links that were already scheduled for download
|
||||
(i.e. all other 'application/' link download will be aborted)</td>
|
||||
(i.e. the download will be aborted)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><tt>-mime:video/* -mime:audio/*</tt></td>
|
||||
<td>This will refuse all audio and video links that were already scheduled for download
|
||||
(i.e. all other 'application/' link download will be aborted)</td>
|
||||
(i.e. the download will be aborted)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><tt>-mime:*/* +mime:text/html +mime:image/*</tt></td>
|
||||
@@ -466,7 +466,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
BIN
html/img/android_experts.png
Normal file
|
After Width: | Height: | Size: 94 KiB |
BIN
html/img/android_finished.png
Normal file
|
After Width: | Height: | Size: 32 KiB |
BIN
html/img/android_import.png
Normal file
|
After Width: | Height: | Size: 50 KiB |
BIN
html/img/android_options.png
Normal file
|
After Width: | Height: | Size: 33 KiB |
BIN
html/img/android_permission.png
Normal file
|
After Width: | Height: | Size: 48 KiB |
BIN
html/img/android_progress.png
Normal file
|
After Width: | Height: | Size: 33 KiB |
BIN
html/img/android_project.png
Normal file
|
After Width: | Height: | Size: 30 KiB |
BIN
html/img/android_scanrules.png
Normal file
|
After Width: | Height: | Size: 41 KiB |
BIN
html/img/android_startup.png
Normal file
|
After Width: | Height: | Size: 36 KiB |
BIN
html/img/android_url.png
Normal file
|
After Width: | Height: | Size: 33 KiB |
@@ -111,6 +111,15 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li><a href="shelldoc.html">WinHTTrack/WebHTTrack (GUI version for Windows or Linux/Unix)</a></li>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix relese (WebHTTrack)<br>
|
||||
<br>
|
||||
<li><a href="android.html">HTTrack on Android</a></li>
|
||||
<br>Step-by-step guide for the Android app<br>
|
||||
<br>
|
||||
<li><a href="cmddoc.html">Command-line version</a></li>
|
||||
<br>How to run HTTrack from the command line, and where to find every option<br>
|
||||
<br>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
<br>A task-oriented guide to the command line: common tasks, recipes, and the defaults that surprise newcomers<br>
|
||||
<br>
|
||||
<li><a href="fcguide.html">HTTrack Users Guide By Fred Cohen</a></li>
|
||||
<br>A tutorial that describes all command-line options, for Linux and Windows users<br>
|
||||
<br>
|
||||
@@ -142,7 +151,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ You may also want to check the <tt>httrack.c</tt> and <tt>httrack.h<tt> files to
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -105,238 +105,14 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li>Filters: <a href="filters.html">how to use them</a></li>
|
||||
<br><small>Here you can find informations on filters: how to accept all gif files in a mirror, for example</small>
|
||||
<br><br>
|
||||
<li>List of options</li>
|
||||
</ul>
|
||||
</ul>
|
||||
|
||||
<tt>
|
||||
<pre>
|
||||
<p>The complete, current list of command-line options is generated from the
|
||||
program itself, in <a href="httrack.man.html">the httrack manual page</a>.</p>
|
||||
|
||||
w mirror with automatic wizard
|
||||
This is the default scanning option, the engine automatically scans links according to the default options, and filters defined. It does not prompt a message when a "foreign" link is reached.
|
||||
<p>WinHTTrack and WebHTTrack offer the same options through their graphical option
|
||||
panels, described in the <a href="step9.html">step-by-step guide</a>.</p>
|
||||
|
||||
W semi-automatic mirror with help-wizard (asks questions)
|
||||
This option lets the engine ask the user if a link must be mirrored or not, when a new web has been found.
|
||||
|
||||
g just get files (saved in the current directory)
|
||||
This option forces the engine not to scan the files indicated - i.e. the engine only gets the files indicated.
|
||||
|
||||
i continue an interrupted mirror using the cache
|
||||
This option indicates to the engine that a mirror must be updated or continued.
|
||||
|
||||
rN recurse get with limited link depth of N
|
||||
This option sets the maximum recurse level. Default is infinite (the engine "knows" that it should not go out of current domain)
|
||||
|
||||
a stay on the same address
|
||||
This is the default primary scanning option, the engine does not go out of domains without permissions (filters, for example)
|
||||
|
||||
d stay on the same principal domain
|
||||
This option lets the engine go on all sites that exist on the same principal domain.
|
||||
Example: a link located at www.example.com that goes to members.example.com will be followed.
|
||||
|
||||
l stay on the same location (.com, etc.)
|
||||
This option lets the engine go on all sites that exist on the same location.
|
||||
Example: a link located at www.example.com that goes to www.anyotherweb.com will be followed.
|
||||
Warning: this is a potentially dangerous option, limit the recurse depth with r option.
|
||||
|
||||
e go everywhere on the web
|
||||
This option lets the engine go on any sites.
|
||||
Example: a link located at www.example.com that goes to www.anyotherweb.org will be followed.
|
||||
Warning: this is a potentially dangerous option, limit the recurse depth with r option.
|
||||
|
||||
n get non-html files 'near' an html file (ex: an image located outside)
|
||||
This option lets the engine catch all files that have references on a page, but that exist outside the web site.
|
||||
Example: List of ZIP files links on a page.
|
||||
|
||||
t test all URLs (even forbidden ones)
|
||||
This option lets the engine test all links that are not caught.
|
||||
Example: to test broken links in a site
|
||||
|
||||
x replace external html links by error pages
|
||||
This option tells the engine to rewrite all links not taken into warning pages.
|
||||
Example: to browse offline a site, and to warn people that they must be online if they click to external links.
|
||||
|
||||
sN follow robots.txt and meta robots tags
|
||||
This option sets the way the engine treats "robots.txt" files. This file is often set by webmasters to avoir cgi-bin directories, or other irrevelant pages.
|
||||
Values:
|
||||
s0 Do not take robots.txt rules
|
||||
s1 Follow rules, if compatible with internal filters
|
||||
s2 Always follow site's rules
|
||||
|
||||
bN accept cookies in cookies.txt
|
||||
This option activates or unactivates the cookie
|
||||
b0 do not accept cookies
|
||||
b1 accept cookies
|
||||
|
||||
S stay on the same directory
|
||||
This option asks the engine to stay on the same folder level.
|
||||
Example: A link in /index.html that points to /sub/other.html will not be followed
|
||||
|
||||
D can only go down into subdirs
|
||||
This is the default option, the engine can go everywhere on the same directoy, or in lower structures
|
||||
|
||||
U can only go to upper directories
|
||||
This option asks the engine to stay on the same folder level or in upper structures
|
||||
|
||||
B can both go up&down into the directory structure
|
||||
This option lets the engine to go in any directory level
|
||||
|
||||
Y mirror ALL links located in the first level pages (mirror links)
|
||||
This option is activated for the links typed in the command line
|
||||
Example: if you have a list of web sites in www.asitelist.com/index.html, then all these sites will be mirrored
|
||||
|
||||
NN name conversion type (0 *original structure 1,2,3 html/data in one directory)
|
||||
N0 Site-structure (default)
|
||||
N1 Html in web/, images/other files in web/images/
|
||||
N2 Html in web/html, images/other in web/images
|
||||
N3 Html in web/, images/other in web/
|
||||
N4 Html in web/, images/other in web/xxx, where xxx is the file extension (all gif will be placed onto web/gif, for example)
|
||||
N5 Images/other in web/xxx and Html in web/html
|
||||
|
||||
N99 All files in web/, with random names (gadget !)
|
||||
|
||||
N100 Site-structure, without www.domain.xxx/
|
||||
N101 Identical to N1 except that "web" is replaced by the site's name
|
||||
N102 Identical to N2 except that "web" is replaced by the site's name
|
||||
N103 Identical to N3 except that "web" is replaced by the site's name
|
||||
N104 Identical to N4 except that "web" is replaced by the site's name
|
||||
N105 Identical to N5 except that "web" is replaced by the site's name
|
||||
N199 Identical to N99 except that "web" is replaced by the site's name
|
||||
|
||||
N1001 Identical to N1 except that there is no "web" directory
|
||||
N1002 Identical to N2 except that there is no "web" directory
|
||||
N1003 Identical to N3 except that there is no "web" directory (option set for g option)
|
||||
N1004 Identical to N4 except that there is no "web" directory
|
||||
N1005 Identical to N5 except that there is no "web" directory
|
||||
N1099 Identical to N99 except that there is no "web" directory
|
||||
|
||||
LN long names
|
||||
L0 Filenames and directory names are limited to 8 characters + 3 for extension
|
||||
L1 No restrictions (default)
|
||||
|
||||
K keep original links (e.g. http://www.adr/link) (K0 *relative link)
|
||||
This option has only been kept for compatibility reasons
|
||||
|
||||
pN priority mode:
|
||||
p0 just scan, don't save anything (for checking links)
|
||||
p1 save only html files
|
||||
p2 save only non html files
|
||||
p3 save all files
|
||||
p7 get html files before, then treat other files
|
||||
|
||||
cN number of multiple connections (*c8)
|
||||
Set the numer of multiple simultaneous connections
|
||||
|
||||
O path for mirror/logfiles+cache (-O path_mirror[,path_cache_and_logfiles])
|
||||
This option define the path for mirror and log files
|
||||
Example: -P "/user/webs","/user/logs"
|
||||
|
||||
P proxy use (-P proxy:port or -P user:pass@proxy:port)
|
||||
This option define the proxy used in this mirror
|
||||
Example: -P proxy.myhost.com:8080
|
||||
|
||||
F user-agent field (-F \"user-agent name\
|
||||
This option define the user-agent field
|
||||
Example: -F "Mozilla/4.5 (compatible; HTTrack 1.2x; Windows 98)"
|
||||
|
||||
mN maximum file length for a non-html file
|
||||
This option define the maximum size for non-html files
|
||||
Example: -m100000
|
||||
|
||||
mN,N' for non html (N) and html (N')
|
||||
This option define the maximum size for non-html files and html-files
|
||||
Example: -m100000,250000
|
||||
|
||||
MN maximum overall size that can be uploaded/scanned
|
||||
This option define the maximum amount of bytes that can be downloaded
|
||||
Example: -M1000000
|
||||
|
||||
EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)
|
||||
This option define the maximum time that the mirror can last
|
||||
Example: -E3600
|
||||
|
||||
AN maximum transfer rate in bytes/seconds (1000=1kb/s max)
|
||||
This option define the maximum transfer rate
|
||||
Example: -A2000
|
||||
|
||||
GN pause transfer if N bytes reached, and wait until lock file is deleted
|
||||
This option asks the engine to pause every time N bytes have been transferred, and restarts when the lock file "hts-pause.lock" is being deleted
|
||||
Example: -G20000000
|
||||
|
||||
u check document type if unknown (cgi,asp..)
|
||||
This option define the way the engine checks the file type
|
||||
u0 do not check
|
||||
u1 check but /
|
||||
u2 check always
|
||||
|
||||
RN number of retries, in case of timeout or non-fatal errors (*R0)
|
||||
This option sets the maximum number of tries that can be processed for a file
|
||||
|
||||
o *generate output html file in case of error (404..) (o0 don't generate)
|
||||
This option define whether the engine has to generate html output file or not if an error occurred
|
||||
|
||||
TN timeout, number of seconds after a non-responding link is shutdown
|
||||
This option define the timeout
|
||||
Example: -T120
|
||||
|
||||
JN traffic jam control, minimum transfert rate (bytes/seconds) tolerated for a link
|
||||
This option define the minimum transfer rate
|
||||
Example: -J200
|
||||
|
||||
HN host is abandoned if: 0=never, 1=timeout, 2=slow, 3=timeout or slow
|
||||
This option define whether the engine has to abandon a host if a timeout/"too slow" error occurred
|
||||
|
||||
&P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
|
||||
This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
|
||||
|
||||
j *parse scripts (j0 don't parse)
|
||||
This option defines whether the engine has to parse scripts or not to catch included files
|
||||
|
||||
I *make an index (I0 don't make)
|
||||
This option define whether the engine has to generate an index.html on the top directory
|
||||
|
||||
X *delete old files after update (X0 keep delete)
|
||||
This option define whether the engine has to delete locally, after an update, files that have been deleted in the remote mirror, or that have been excluded
|
||||
|
||||
C *create/use a cache for updates and retries (C0 no cache)
|
||||
This option define whether the engine has to generate a cache for retries and updates or not
|
||||
|
||||
k store all files in cache (not useful if files on disk)
|
||||
This option define whether the engine has to store all files in cache or not
|
||||
|
||||
V execute system command after each files ($0 is the filename: -V \"rm \\$0\
|
||||
This option lets the engine execute a command for each file saved on disk
|
||||
|
||||
q quiet mode (no questions)
|
||||
Do not ask questions (for example, for confirm an option)
|
||||
|
||||
Q log quiet mode (no log)
|
||||
Do not generate log files
|
||||
|
||||
v verbose screen mode
|
||||
Log files are printed in the screen
|
||||
|
||||
f *log file mode
|
||||
Log files are generated into two log files
|
||||
|
||||
z extra infos log
|
||||
Add more informations on log files
|
||||
|
||||
Z debug log
|
||||
Add debug informations on log files
|
||||
|
||||
|
||||
--mirror <URLs> *make a mirror of site(s)
|
||||
--get <URLs> get the files indicated, do not seek other URLs
|
||||
--mirrorlinks <URLs> test links in pages (identical to -Y)
|
||||
--testlinks <URLs> test links in pages
|
||||
--spider <URLs> spider site(s), to test links (reports Errors & Warnings)
|
||||
--update <URLs> update a mirror, without confirmation
|
||||
--skeleton<URLs> make a mirror, but gets only html files
|
||||
|
||||
--http10 force http/1.0 requests when possible
|
||||
|
||||
</pre>
|
||||
</tt>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
|
||||
@@ -352,7 +128,7 @@ Add debug informations on log files
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ downloads. HTTrack is fully configurable, and has an integrated help system.
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -286,7 +286,7 @@ module exit point
|
||||
the function name and prototype MUST match this prototype
|
||||
*/
|
||||
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
|
||||
fprintf(stder, "Module unplugged");
|
||||
fprintf(stderr, "Module unplugged");
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
@@ -331,7 +331,7 @@ httrack --wrapper mylibrary,myparameter-string http://www.example.com
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -204,7 +204,7 @@ Below additional function names that can be defined inside the optional libhttra
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -250,7 +250,7 @@ Script example:
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ ${LANG_K3} : ${HTTRACK_WEB}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -159,7 +159,7 @@ ${do:end-if}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ ${error}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ ${do:loadhash}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -143,7 +143,7 @@ ${path}/${projname}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ ${do:end-if}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ ${LANG_THANKYOU}!
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -158,7 +158,7 @@ ${do:end-if}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ ${LANG_IOPT10}:
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -250,7 +250,7 @@ ${LANG_W3}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@ ${listid:build:LISTDEF_3}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -140,7 +140,7 @@ ${do:output-mode:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -191,7 +191,7 @@ ${listid:travel3:LISTDEF_11}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -184,7 +184,7 @@ ${LANG_I46}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -228,7 +228,7 @@ ${LANG_I64b}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -156,7 +156,7 @@ ${LANG_I43b}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -148,7 +148,7 @@ ${LANG_B13}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -212,7 +212,7 @@ ${LANG_STRIPQUERY}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -166,7 +166,7 @@ ${listid:logtype:LISTDEF_9}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -204,7 +204,7 @@ ${LANG_H20} ${info.currentjob}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -283,7 +283,7 @@ ${do:end-if:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -196,7 +196,7 @@ ${do:output-mode:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -324,7 +324,7 @@ ${do:output-mode:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ You may encounter minor differences (in the display, or in various options) betw
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -128,7 +128,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -143,7 +143,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -157,7 +157,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -128,7 +128,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -150,7 +150,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ In this case, HTTrack won't check the type, because it has learned that "foo" is
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -181,7 +181,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -165,7 +165,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -162,7 +162,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -156,7 +156,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ pt:7
|
||||
ro:25
|
||||
ru:8
|
||||
sk:20
|
||||
si:24
|
||||
sl:24
|
||||
sv:17
|
||||
tr:10
|
||||
uk:22
|
||||
|
||||
@@ -930,7 +930,6 @@ Server terminated
|
||||
Palvelin lopetettu
|
||||
A fatal error has occurred during this mirror
|
||||
Tällä peilillä tapahtui vakava virhe
|
||||
|
||||
Proxy type:
|
||||
Välityspalvelimen tyyppi:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
|
||||
@@ -374,7 +374,7 @@ Create error logging and report files
|
||||
Crea file di log per segnalare errori e informazioni
|
||||
Generate DOS 8-3 filenames ONLY
|
||||
Genera solo nomi di file in formato 8.3
|
||||
Generate ISO 9660 filenames ONLY for CDROM medias
|
||||
Generate ISO9660 filenames ONLY for CDROM medias
|
||||
Genera nomi di file in formato ISO9660 per i CDROM
|
||||
Do not create HTML error pages
|
||||
Non generare pagine d'errore HTML
|
||||
|
||||
@@ -918,7 +918,7 @@ normal\nextended\ndebug
|
||||
normal\nextendido\ncorrigir
|
||||
Download web site(s)\nDownload web site(s) + questions\nGet individual files\nDownload all sites in pages (multiple mirror)\nTest links in pages (bookmark test)\n* Continue interrupted download\n* Update existing download
|
||||
Copiar site(s) da Web\nCopiar site(s) interativos da Web (perguntas)\nReceber arquivos específicos\nCopiar todas as páginas do site (alternação múltipla)\nTestar links nas páginas (testar indicador)\n* Retomar download interrompido\n* Atualizar download existente
|
||||
Relative URL / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
|
||||
Relative URI / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
|
||||
URI Relativa / URL Absoluta (padrão)\nURL Absoluta / URL Absoluta\nURI Absoluta / URL Absoluta\nURL Original / URL Original
|
||||
Open Source offline browser
|
||||
Abrir origem offline no navegador
|
||||
|
||||
@@ -3,7 +3,7 @@ Slovenian
|
||||
LANGUAGE_FILE
|
||||
Slovenian
|
||||
LANGUAGE_ISO
|
||||
si
|
||||
sl
|
||||
LANGUAGE_AUTHOR
|
||||
Jadran Rudec,iur.\r\njrudec@email.si \r\n
|
||||
LANGUAGE_CHARSET
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "14 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "16 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -165,7 +165,7 @@ pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\
|
||||
.IP \-cN
|
||||
number of multiple connections (*c8) (\-\-sockets[=N])
|
||||
.IP \-TN
|
||||
timeout, number of seconds after a non\-responding link is shutdown (\-\-timeout[=N])
|
||||
timeout, number of seconds after a non\-responding link is shutdown; also bounds host name resolution (\-\-timeout[=N])
|
||||
.IP \-RN
|
||||
number of retries, in case of timeout or non\-fatal errors (*R1) (\-\-retries[=N])
|
||||
.IP \-JN
|
||||
|
||||
@@ -46,9 +46,11 @@ htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
|
||||
|
||||
lib_LTLIBRARIES = libhttrack.la
|
||||
|
||||
htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
|
||||
htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h \
|
||||
htsurlport.c htsurlport.h
|
||||
proxytrack_SOURCES = proxy/main.c \
|
||||
proxy/proxytrack.c proxy/store.c \
|
||||
htsurlport.c htsurlport.h \
|
||||
coucal/coucal.c htsmd5.c md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c
|
||||
|
||||
@@ -58,7 +60,7 @@ whttrackrun_SCRIPTS = webhttrack
|
||||
libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htscache_selftest.c htsdns_selftest.c htsselftest.c \
|
||||
htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
|
||||
htshelp.c htslib.c htscoremain.c \
|
||||
htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
@@ -69,7 +71,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h \
|
||||
htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
|
||||
htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
|
||||
htshelp.h htsindex.h htslib.h htsmd5.h \
|
||||
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htsproxy.h htszlib.h \
|
||||
|
||||
@@ -567,24 +567,29 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Get home directory, '.' if failed */
|
||||
/* Get home directory, '.' if unset or empty */
|
||||
/* example: /home/smith */
|
||||
const char *hts_gethome(void) {
|
||||
const char *home = getenv("HOME");
|
||||
|
||||
if (home)
|
||||
return home;
|
||||
else
|
||||
return ".";
|
||||
/* An empty $HOME would expand ~/foo into the absolute /foo */
|
||||
return strnotempty(home) ? home : ".";
|
||||
}
|
||||
|
||||
/* Convert ~/foo into /home/smith/foo */
|
||||
/* Convert ~/foo into /home/smith/foo (~user/ left alone: no getpwnam here) */
|
||||
void expand_home(String * str) {
|
||||
if (StringSub(*str, 1) == '~') {
|
||||
if (StringNotEmpty(*str) && StringSub(*str, 0) == '~' &&
|
||||
(StringLength(*str) == 1 || StringSub(*str, 1) == '/')) {
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
const char *const home = hts_gethome();
|
||||
const size_t homelen = strlen(home);
|
||||
const size_t taillen = StringLength(*str) - 1;
|
||||
|
||||
strcpybuff(tempo, hts_gethome());
|
||||
strcatbuff(tempo, StringBuff(*str) + 1);
|
||||
StringCopy(*str, tempo);
|
||||
/* Leave untouched rather than abort() in strcatbuff on a huge $HOME */
|
||||
if (taillen < sizeof(tempo) && homelen < sizeof(tempo) - taillen) {
|
||||
strcpybuff(tempo, home);
|
||||
strcatbuff(tempo, StringBuff(*str) + 1);
|
||||
StringCopy(*str, tempo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2816,6 +2816,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (SSL_set_fd(back[i].r.ssl_con, (int) back[i].r.soc) == 1) {
|
||||
SSL_set_connect_state(back[i].r.ssl_con);
|
||||
back[i].status = STATUS_SSL_WAIT_HANDSHAKE; /* handshake wait */
|
||||
// the handshake gets its own timeout window, as connect does
|
||||
if (back[i].timeout > 0)
|
||||
back[i].timeout_refresh = time_local();
|
||||
} else
|
||||
back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE;
|
||||
} else
|
||||
@@ -2875,6 +2878,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
#if HTS_USEOPENSSL
|
||||
else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE) { // wait for SSL handshake
|
||||
// a peer that never speaks TLS must be reaped by --timeout too (#607)
|
||||
if (!gestion_timeout)
|
||||
if (back[i].timeout > 0)
|
||||
gestion_timeout = 1;
|
||||
|
||||
/* SSL mode */
|
||||
if (back[i].r.ssl) {
|
||||
int conn_code;
|
||||
@@ -4265,6 +4273,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
strcpybuff(back[i].r.msg, "Connect Time Out");
|
||||
else if (back[i].status == STATUS_WAIT_DNS)
|
||||
strcpybuff(back[i].r.msg, "DNS Time Out");
|
||||
else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE)
|
||||
strcpybuff(back[i].r.msg, "SSL/TLS Handshake Time Out");
|
||||
else
|
||||
strcpybuff(back[i].r.msg, "Receive Time Out");
|
||||
back[i].status = STATUS_READY; // terminé
|
||||
|
||||
@@ -373,7 +373,8 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
|
||||
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset) {
|
||||
const UINT cp = hts_getCodepage(charset);
|
||||
|
||||
return hts_convertStringCPToUTF8(s, size, cp);
|
||||
@@ -554,7 +555,8 @@ static char *hts_convertStringCharset(const char *s, size_t size,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
|
||||
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset) {
|
||||
/* Empty string ? */
|
||||
if (size == 0) {
|
||||
return strdup("");
|
||||
|
||||
@@ -51,8 +51,8 @@ typedef unsigned int hts_UCS4;
|
||||
* Convert the string "s" from charset "charset" to UTF-8.
|
||||
* Return NULL upon error.
|
||||
**/
|
||||
extern char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset);
|
||||
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset);
|
||||
|
||||
/**
|
||||
* Convert the string "s" from UTF-8 to charset "charset".
|
||||
|
||||
@@ -113,6 +113,7 @@ HTSEXT_API int hts_main(int argc, char **argv) {
|
||||
}
|
||||
|
||||
static int hts_main_internal(int argc, char **argv, httrackp * opt);
|
||||
static hts_boolean cmdl_shortopt_has(const char *s, char c);
|
||||
|
||||
// Main, récupère les paramètres et appelle le robot
|
||||
HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt) {
|
||||
@@ -304,12 +305,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
hts_get_version_info(opt));
|
||||
return 0;
|
||||
} else {
|
||||
if (strncmp(tmp_argv[0], "--", 2)) { /* pas */
|
||||
if ((strchr(tmp_argv[0], 'q') != NULL))
|
||||
opt->quiet = 1; // ne pas poser de questions! (nohup par exemple)
|
||||
if ((strchr(tmp_argv[0], 'i') != NULL)) { // doit.log!
|
||||
argv_url = -1; /* forcer */
|
||||
opt->quiet = 1;
|
||||
if (strncmp(tmp_argv[0], "--", 2)) { /* not a long option */
|
||||
if (cmdl_shortopt_has(tmp_argv[0], 'q'))
|
||||
opt->quiet = HTS_TRUE; // never ask questions (nohup)
|
||||
if (cmdl_shortopt_has(tmp_argv[0], 'i')) { // doit.log!
|
||||
argv_url = -1;
|
||||
opt->quiet = HTS_TRUE;
|
||||
}
|
||||
} else if (strcmp(tmp_argv[0] + 2, "quiet") == 0) {
|
||||
opt->quiet = 1; // ne pas poser de questions! (nohup par exemple)
|
||||
@@ -424,25 +425,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
|
||||
} // for
|
||||
|
||||
// Convert path to UTF-8
|
||||
#ifdef _WIN32
|
||||
{
|
||||
char *const path =
|
||||
hts_convertStringSystemToUTF8(StringBuff(opt->path_html),
|
||||
(int) StringLength(opt->path_html));
|
||||
if (path != NULL) {
|
||||
StringCopy(opt->path_html_utf8, path);
|
||||
free(path);
|
||||
} else {
|
||||
StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
|
||||
StringLength(opt->path_html));
|
||||
}
|
||||
}
|
||||
#else
|
||||
// Assume UTF-8 filesystem.
|
||||
// path_html is already UTF-8 (argv is UTF-8 on Windows via
|
||||
// hts_argv_utf8), so no re-encoding.
|
||||
StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
|
||||
StringLength(opt->path_html));
|
||||
#endif
|
||||
|
||||
/* if doit.log exists, or if new URL(s) defined,
|
||||
then DO NOT load standard config files */
|
||||
@@ -2382,30 +2368,28 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
opt->errlog = stderr;
|
||||
} else if (httrack_logmode >= 2) {
|
||||
// deux fichiers log
|
||||
structcheck(StringBuff(opt->path_log));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"));
|
||||
// path_log holds UTF-8 bytes (argv is UTF-8): the ANSI file calls would
|
||||
// read them as the codepage and drop the logs into a mangled twin (#630).
|
||||
structcheck_utf8(StringBuff(opt->path_log));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt"));
|
||||
|
||||
/* Check FS directory structure created */
|
||||
structcheck(StringBuff(opt->path_log));
|
||||
structcheck_utf8(StringBuff(opt->path_log));
|
||||
|
||||
opt->log =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"),
|
||||
"w");
|
||||
opt->log = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt"),
|
||||
"w");
|
||||
if (httrack_logmode == 2)
|
||||
opt->errlog =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"),
|
||||
"w");
|
||||
opt->errlog = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt"),
|
||||
"w");
|
||||
else
|
||||
opt->errlog = opt->log;
|
||||
if (opt->log == NULL) {
|
||||
@@ -2485,9 +2469,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
}
|
||||
} */
|
||||
|
||||
// vérifier existence de la structure
|
||||
structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "/"));
|
||||
structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "/"));
|
||||
// vérifier existence de la structure (path_html/path_log are UTF-8, use
|
||||
// the UTF-8 mkdir path)
|
||||
structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_html), "/"));
|
||||
structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "/"));
|
||||
|
||||
// reprise/update
|
||||
if (opt->cache) {
|
||||
@@ -2807,6 +2794,32 @@ int check_path(String * s, char *defaultname) {
|
||||
return return_value;
|
||||
}
|
||||
|
||||
/* Does the short-option cluster s carry c from the main option set (-i, -iC2,
|
||||
-%Mi)? Walked as the parser does below: %, &, @ and # each take the letter
|
||||
after them into another set, so the i of -%i is not the main-set -i. */
|
||||
static hts_boolean cmdl_shortopt_has(const char *s, char c) {
|
||||
const char *com;
|
||||
|
||||
if (s[0] != '-' || s[1] == '-')
|
||||
return HTS_FALSE;
|
||||
for (com = s + 1; *com != '\0'; com++) {
|
||||
switch (*com) {
|
||||
case '%':
|
||||
case '&':
|
||||
case '@':
|
||||
case '#':
|
||||
if (*(com + 1) != '\0')
|
||||
com++; /* skip the other set's letter */
|
||||
break;
|
||||
default:
|
||||
if (*com == c)
|
||||
return HTS_TRUE;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
// détermine si l'argument est une option
|
||||
int cmdl_opt(char *s) {
|
||||
if (s[0] == '-') { // c'est peut être une option
|
||||
|
||||
@@ -63,9 +63,14 @@ typedef struct mock_host {
|
||||
int gai_err; /* non-zero: getaddrinfo returns this */
|
||||
int naddr;
|
||||
mock_addr addr[6];
|
||||
int calls; /* times the backend resolved this host */
|
||||
int calls; /* times the backend resolved this host */
|
||||
int slow_ms; /* non-zero: block this long, as a black-hole resolver would */
|
||||
} mock_host;
|
||||
|
||||
/* Long enough to outlast the 1s --timeout the bounded resolve is checked
|
||||
against, short enough to keep the self-test quick. */
|
||||
#define MOCK_SLOW_MS 3000
|
||||
|
||||
static mock_host mock_hosts[] = {
|
||||
{"v4only.test", 0, 1, {{AF_INET, {1, 2, 3, 4}}}, 0},
|
||||
{"v6only.test", 0, 1, {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 1}}}, 0},
|
||||
@@ -95,8 +100,15 @@ static mock_host mock_hosts[] = {
|
||||
{AF_INET, {10, 0, 0, 6}}},
|
||||
0},
|
||||
{"nodns.test", EAI_NONAME, 0, {{0}}, 0},
|
||||
/* resolves, but only well after --timeout: the #606 wedge */
|
||||
{"slow.test", 0, 1, {{AF_INET, {127, 0, 0, 9}}}, 0, MOCK_SLOW_MS},
|
||||
};
|
||||
|
||||
/* Serializes mock_host bookkeeping: a timed-out resolve is abandoned, so its
|
||||
worker is still inside the backend while the test reads the counters. */
|
||||
static htsmutex mock_lock = HTSMUTEX_INIT;
|
||||
static int mock_finished = 0; /* backend calls that have returned */
|
||||
|
||||
static mock_host *mock_find(const char *name) {
|
||||
for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++) {
|
||||
if (strcmp(mock_hosts[i].name, name) == 0)
|
||||
@@ -106,8 +118,34 @@ static mock_host *mock_find(const char *name) {
|
||||
}
|
||||
|
||||
static void mock_reset_calls(void) {
|
||||
hts_mutexlock(&mock_lock);
|
||||
for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++)
|
||||
mock_hosts[i].calls = 0;
|
||||
mock_finished = 0;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
}
|
||||
|
||||
static int mock_read_calls(const char *name) {
|
||||
int calls;
|
||||
|
||||
hts_mutexlock(&mock_lock);
|
||||
calls = mock_find(name)->calls;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
return calls;
|
||||
}
|
||||
|
||||
/* Wait for n backend calls to return, ordering their writes against ours. */
|
||||
static void mock_wait_finished(int n) {
|
||||
for (;;) {
|
||||
hts_boolean done;
|
||||
|
||||
hts_mutexlock(&mock_lock);
|
||||
done = (mock_finished >= n) ? HTS_TRUE : HTS_FALSE;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
if (done)
|
||||
break;
|
||||
Sleep(10);
|
||||
}
|
||||
}
|
||||
|
||||
/* Build one addrinfo node owning its sockaddr (freed by mock_freeaddrinfo). */
|
||||
@@ -133,10 +171,10 @@ static struct addrinfo *mock_mkai(const mock_addr *a) {
|
||||
return ai;
|
||||
}
|
||||
|
||||
static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
const char *service,
|
||||
const struct addrinfo *hints,
|
||||
struct addrinfo **res) {
|
||||
static int HTS_RESOLVER_CALL mock_getaddrinfo_(const char *node,
|
||||
const char *service,
|
||||
const struct addrinfo *hints,
|
||||
struct addrinfo **res) {
|
||||
mock_host *const h = mock_find(node);
|
||||
const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
|
||||
struct addrinfo *head = NULL, *tail = NULL;
|
||||
@@ -145,7 +183,11 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
*res = NULL;
|
||||
if (h == NULL)
|
||||
return EAI_NONAME;
|
||||
hts_mutexlock(&mock_lock);
|
||||
h->calls++; /* a real backend hit; a cached host skips this */
|
||||
hts_mutexrelease(&mock_lock);
|
||||
if (h->slow_ms != 0)
|
||||
Sleep(h->slow_ms);
|
||||
if (h->gai_err != 0)
|
||||
return h->gai_err;
|
||||
for (int i = 0; i < h->naddr; i++) {
|
||||
@@ -165,6 +207,18 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
|
||||
const char *service,
|
||||
const struct addrinfo *hints,
|
||||
struct addrinfo **res) {
|
||||
const int ret = mock_getaddrinfo_(node, service, hints, res);
|
||||
|
||||
hts_mutexlock(&mock_lock);
|
||||
mock_finished++;
|
||||
hts_mutexrelease(&mock_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void HTS_RESOLVER_CALL mock_freeaddrinfo(struct addrinfo *res) {
|
||||
while (res != NULL) {
|
||||
struct addrinfo *const next = res->ai_next;
|
||||
@@ -348,6 +402,50 @@ int dns_selftests(httrackp *opt) {
|
||||
deletesoc(s);
|
||||
}
|
||||
|
||||
/* A URL port outside 1..65535 must refuse the link, not fold into range and
|
||||
connect elsewhere (#614). *addr_count discriminates: 0 only if refused
|
||||
before the resolve, still 2 for one merely truncated or defaulted. */
|
||||
{
|
||||
/* an empty "dual.test:" means the default port (WHATWG, curl): keep it */
|
||||
static const char *const good[] = {"dual.test:1", "dual.test:80",
|
||||
"dual.test:8080", "dual.test:65535",
|
||||
"dual.test:080", "dual.test:"};
|
||||
/* 65616 and 4294967376 are load-bearing: both wrap to a plausible 80 */
|
||||
static const char *const bad[] = {
|
||||
"dual.test:0", "dual.test:65536", "dual.test:65616",
|
||||
"dual.test:99999", "dual.test:2147483648", "dual.test:4294967296",
|
||||
"dual.test:4294967376", "dual.test:-1", "dual.test:-23437",
|
||||
"dual.test:80x", "dual.test:+80", "dual.test: 80",
|
||||
"dual.test:0x50"};
|
||||
size_t k;
|
||||
|
||||
for (k = 0; k < sizeof(good) / sizeof(good[0]); k++) {
|
||||
htsblk r;
|
||||
int count = -1;
|
||||
T_SOC s;
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
s = newhttp_addr(opt, good[k], &r, -1, 0, 0, &count);
|
||||
CHECK(count == 2); /* accepted: reached the resolve */
|
||||
if (s != INVALID_SOCKET)
|
||||
deletesoc(s);
|
||||
}
|
||||
|
||||
for (k = 0; k < sizeof(bad) / sizeof(bad[0]); k++) {
|
||||
htsblk r;
|
||||
int count = -1;
|
||||
T_SOC s;
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
s = newhttp_addr(opt, bad[k], &r, -1, 0, 0, &count);
|
||||
CHECK(s == INVALID_SOCKET);
|
||||
CHECK(count == 0); /* refused before resolving, not a failed connect */
|
||||
CHECK(strstr(r.msg, "Invalid port") != NULL);
|
||||
if (s != INVALID_SOCKET)
|
||||
deletesoc(s);
|
||||
}
|
||||
}
|
||||
|
||||
/* Connect-fallback decision (consumer of the multi-address list): when a
|
||||
stuck connect should abandon the current address for the next one. */
|
||||
{
|
||||
@@ -370,6 +468,91 @@ int dns_selftests(httrackp *opt) {
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* Probes how long acquiring opt->state.lock takes while a resolve is in
|
||||
flight. hts_has_stopped() takes that same lock and mutates nothing, and the
|
||||
API promises it stays callable from another thread during a mirror. */
|
||||
typedef struct lock_probe {
|
||||
httrackp *opt;
|
||||
htsmutex lock;
|
||||
hts_boolean done;
|
||||
TStamp blocked_ms;
|
||||
} lock_probe;
|
||||
|
||||
static void lock_probe_thread(void *arg) {
|
||||
lock_probe *const p = (lock_probe *) arg;
|
||||
TStamp start;
|
||||
|
||||
Sleep(MOCK_SLOW_MS / 10); /* let the resolve get under way first */
|
||||
start = mtime_local();
|
||||
(void) hts_has_stopped(p->opt);
|
||||
hts_mutexlock(&p->lock);
|
||||
p->blocked_ms = mtime_local() - start;
|
||||
p->done = HTS_TRUE;
|
||||
hts_mutexrelease(&p->lock);
|
||||
}
|
||||
|
||||
int dns_timeout_selftests(httrackp *opt) {
|
||||
SOCaddr addrs[HTS_MAXADDRNUM];
|
||||
const char *err = NULL;
|
||||
lock_probe probe;
|
||||
TStamp start, elapsed;
|
||||
int count;
|
||||
|
||||
failures = 0;
|
||||
hts_dns_set_resolver_backend(&mock_backend);
|
||||
IPV6_resolver = 0;
|
||||
mock_reset_calls();
|
||||
opt->timeout = 1; /* the bound under test */
|
||||
|
||||
memset(&probe, 0, sizeof(probe));
|
||||
probe.opt = opt;
|
||||
probe.lock = HTSMUTEX_INIT;
|
||||
hts_mutexinit(&probe.lock);
|
||||
CHECK(hts_newthread(lock_probe_thread, &probe) == 0);
|
||||
|
||||
start = mtime_local();
|
||||
count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
|
||||
elapsed = mtime_local() - start;
|
||||
|
||||
/* the resolve returns on opt->timeout, not when the resolver deigns to
|
||||
answer: this is what lets --max-time and --timeout fire (#606). The bound
|
||||
is derived from opt->timeout, never from the mock's sleep, or a resolve
|
||||
that ignored opt->timeout would still pass under the mock. */
|
||||
CHECK(elapsed < (TStamp) opt->timeout * 1000 + 500);
|
||||
CHECK(count == 0); /* a timeout is reported as "does not resolve" */
|
||||
|
||||
/* state.lock is not held across the resolve; a concurrent stop query, which
|
||||
the mirror API promises stays live, is not blocked behind it */
|
||||
for (;;) {
|
||||
hts_boolean done;
|
||||
TStamp blocked;
|
||||
|
||||
hts_mutexlock(&probe.lock);
|
||||
done = probe.done;
|
||||
blocked = probe.blocked_ms;
|
||||
hts_mutexrelease(&probe.lock);
|
||||
if (done) {
|
||||
CHECK(blocked < 500);
|
||||
break;
|
||||
}
|
||||
Sleep(20);
|
||||
}
|
||||
hts_mutexfree(&probe.lock);
|
||||
|
||||
/* a timeout is not an answer, so it must not be negative-cached: the host is
|
||||
resolved again rather than written off for the rest of the crawl */
|
||||
CHECK(mock_read_calls("slow.test") == 1);
|
||||
count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
|
||||
CHECK(count == 0);
|
||||
CHECK(mock_read_calls("slow.test") == 2); /* re-resolved, not cached */
|
||||
|
||||
/* Both resolves were abandoned mid-backend; wait for their workers to leave
|
||||
it before returning. The backend stays installed: an abandoned worker
|
||||
still reads it (to free its addrinfo) after the last call returns. */
|
||||
mock_wait_finished(2);
|
||||
return failures;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int dns_selftests(httrackp *opt) {
|
||||
|
||||
@@ -46,6 +46,12 @@ typedef struct httrackp httrackp;
|
||||
Returns the number of failed checks (0 == success). */
|
||||
int dns_selftests(httrackp *opt);
|
||||
|
||||
/* Drive a deliberately slow (mock) resolver, asserting that a resolve is
|
||||
bounded by opt->timeout, does not hold opt->state.lock while it runs, and
|
||||
does not cache a timeout as an answer (#606). Takes a few seconds.
|
||||
Returns the number of failed checks (0 == success). */
|
||||
int dns_timeout_selftests(httrackp *opt);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -248,7 +248,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
// port
|
||||
a = strchr(adr, ':'); // port
|
||||
if (a) {
|
||||
sscanf(a + 1, "%d", &port);
|
||||
// folding a nonsense port into 1..65535 fetches one the link never named;
|
||||
// an empty "host:" just means the default (#614)
|
||||
if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Invalid port: %s", a + 1);
|
||||
back->r.statuscode = STATUSCODE_INVALID; // permanent, unlike a DNS miss
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
strncatbuff(_adr, adr, (int) (a - adr));
|
||||
} else
|
||||
strcpybuff(_adr, adr);
|
||||
|
||||
@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
|
||||
configure.ac, decoupled from these). VERSION is the display form, VERSIONID
|
||||
the dotted numeric form, AFF_VERSION the short form shown in footers,
|
||||
LIB_VERSION the data/cache format generation. */
|
||||
#define HTTRACK_VERSION "3.49-12"
|
||||
#define HTTRACK_VERSIONID "3.49.12"
|
||||
#define HTTRACK_VERSION "3.49-13"
|
||||
#define HTTRACK_VERSIONID "3.49.13"
|
||||
#define HTTRACK_AFF_VERSION "3.x"
|
||||
#define HTTRACK_LIB_VERSION "2.0"
|
||||
|
||||
|
||||
@@ -496,8 +496,8 @@ void help(const char *app, int more) {
|
||||
infomsg("");
|
||||
infomsg("Flow control:");
|
||||
infomsg(" cN number of multiple connections (*c8)");
|
||||
infomsg
|
||||
(" TN timeout, number of seconds after a non-responding link is shutdown");
|
||||
infomsg(" TN timeout, number of seconds after a non-responding link is"
|
||||
" shutdown; also bounds host name resolution");
|
||||
infomsg
|
||||
(" RN number of retries, in case of timeout or non-fatal errors (*R1)");
|
||||
infomsg
|
||||
|
||||
216
src/htslib.c
@@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#endif /* _WIN32 */
|
||||
#include <stdarg.h>
|
||||
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <stdarg.h>
|
||||
@@ -2166,15 +2167,18 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
|
||||
#endif
|
||||
|
||||
if (a != NULL) {
|
||||
int i = -1;
|
||||
|
||||
iadr2[0] = '\0';
|
||||
sscanf(a + 1, "%d", &i);
|
||||
if (i != -1) {
|
||||
port = (unsigned short int) i;
|
||||
// folding a nonsense port into 0..65535 crawls one neither the link nor
|
||||
// a port filter named; an empty "host:" just means the default (#614)
|
||||
if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
|
||||
if (retour != NULL) {
|
||||
snprintf(retour->msg, sizeof(retour->msg), "Invalid port: %s",
|
||||
a + 1);
|
||||
}
|
||||
return INVALID_SOCKET;
|
||||
}
|
||||
|
||||
// adresse véritable (sans :xx)
|
||||
iadr2[0] = '\0';
|
||||
// the address itself, without the ":port"
|
||||
strncatbuff(iadr2, iadr, (int) (a - iadr));
|
||||
resolve_host = iadr2;
|
||||
}
|
||||
@@ -3747,6 +3751,16 @@ static int proxy_default_port(const char *arg) {
|
||||
return hts_proxy_is_socks(arg) ? 1080 : 8080;
|
||||
}
|
||||
|
||||
// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
|
||||
// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
|
||||
static int parse_proxy_port(const char *a, const char *arg) {
|
||||
int port;
|
||||
|
||||
if (!hts_parse_url_port(a, &port))
|
||||
return proxy_default_port(arg);
|
||||
return port;
|
||||
}
|
||||
|
||||
void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
const char *authority = strstr(arg, "://");
|
||||
const char *a;
|
||||
@@ -3762,10 +3776,7 @@ void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
while (a > authority && a[-1] != ':' && a[-1] != '@' && a[-1] != ']')
|
||||
a--;
|
||||
if (a > authority && a[-1] == ':') {
|
||||
int p = -1;
|
||||
|
||||
sscanf(a, "%d", &p);
|
||||
*port = (p > 0) ? p : proxy_default_port(arg);
|
||||
*port = parse_proxy_port(a, arg);
|
||||
namelen = (size_t) (a - 1 - arg);
|
||||
} else {
|
||||
*port = proxy_default_port(arg);
|
||||
@@ -5084,50 +5095,164 @@ HTSEXT_API int check_hostname_dns(const char *const hostname) {
|
||||
return hts_dns_resolve_nocache(hostname, &buffer) != NULL;
|
||||
}
|
||||
|
||||
// Needs locking
|
||||
// Internal DNS cache. Fill out[0..count-1] with up to max addresses for _iadr,
|
||||
// resolving (and caching the full list) on a miss. Returns the count.
|
||||
static int hts_dns_resolve_list_(httrackp *opt, const char *_iadr,
|
||||
SOCaddr *const out, const int max,
|
||||
const char **error) {
|
||||
char BIGSTK iadr[HTS_URLMAXSIZE * 2];
|
||||
coucal cache = hts_cache(opt); // le cache dns
|
||||
/* A resolve in flight. Refcounted: a timed-out resolve is abandoned, not
|
||||
cancelled, so the last of caller/worker to leave frees the job. */
|
||||
typedef struct dns_resolve_job {
|
||||
htsmutex lock;
|
||||
int refcount;
|
||||
hts_boolean done;
|
||||
char *hostname;
|
||||
SOCaddr addr[HTS_MAXADDRNUM];
|
||||
int count;
|
||||
const char *error;
|
||||
} dns_resolve_job;
|
||||
|
||||
/* Copy the first min(count, max) addresses of src into dest. */
|
||||
static void dns_copy_addrs(SOCaddr *dest, SOCaddr *src, int count, int max) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < count && i < max; i++)
|
||||
SOCaddr_copy_SOCaddr(dest[i], src[i]);
|
||||
}
|
||||
|
||||
static void dns_job_release(dns_resolve_job *job) {
|
||||
hts_boolean last;
|
||||
|
||||
hts_mutexlock(&job->lock);
|
||||
last = (--job->refcount == 0) ? HTS_TRUE : HTS_FALSE;
|
||||
hts_mutexrelease(&job->lock);
|
||||
if (last) {
|
||||
hts_mutexfree(&job->lock);
|
||||
freet(job->hostname);
|
||||
freet(job);
|
||||
}
|
||||
}
|
||||
|
||||
/* Outlives a timed-out resolve, so it writes only the job: never opt (freed
|
||||
before the thread wait at exit) nor the DNS cache. */
|
||||
static void dns_resolve_thread(void *arg) {
|
||||
dns_resolve_job *const job = (dns_resolve_job *) arg;
|
||||
SOCaddr resolved[HTS_MAXADDRNUM];
|
||||
const char *error = NULL;
|
||||
const int count = hts_dns_resolve_nocache_list(job->hostname, resolved,
|
||||
HTS_MAXADDRNUM, &error);
|
||||
|
||||
hts_mutexlock(&job->lock);
|
||||
dns_copy_addrs(job->addr, resolved, count, HTS_MAXADDRNUM);
|
||||
job->count = count;
|
||||
job->error = error;
|
||||
job->done = HTS_TRUE; /* published last: gates the caller's read of addr[] */
|
||||
hts_mutexrelease(&job->lock);
|
||||
dns_job_release(job);
|
||||
}
|
||||
|
||||
/* Resolve hostname on a worker thread, giving up after timeout seconds.
|
||||
Returns the address count, or -1 on timeout -- distinct from 0 ("does not
|
||||
resolve"), which is a real answer and gets negative-cached. */
|
||||
static int hts_dns_resolve_nocache_list_bounded(const char *hostname,
|
||||
SOCaddr *const out,
|
||||
const int max,
|
||||
const int timeout,
|
||||
const char **error) {
|
||||
dns_resolve_job *job;
|
||||
TStamp deadline;
|
||||
int count = -1;
|
||||
int poll_ms = 1;
|
||||
|
||||
if (timeout <= 0) /* no bound asked for (--timeout 0) */
|
||||
return hts_dns_resolve_nocache_list(hostname, out, max, error);
|
||||
|
||||
job = calloct(1, sizeof(*job));
|
||||
assertf(job != NULL);
|
||||
hts_mutexinit(&job->lock);
|
||||
job->hostname = strdupt(hostname);
|
||||
job->refcount = 2; /* this caller + the worker */
|
||||
if (hts_newthread(dns_resolve_thread, job) != 0) {
|
||||
job->refcount = 1; /* no worker: fall back to resolving inline */
|
||||
dns_job_release(job);
|
||||
return hts_dns_resolve_nocache_list(hostname, out, max, error);
|
||||
}
|
||||
|
||||
deadline = mtime_local() + (TStamp) timeout * 1000;
|
||||
for (;;) {
|
||||
hts_boolean done;
|
||||
|
||||
hts_mutexlock(&job->lock);
|
||||
done = job->done;
|
||||
if (done) {
|
||||
count = job->count;
|
||||
dns_copy_addrs(out, job->addr, count, max);
|
||||
if (error != NULL)
|
||||
*error = job->error;
|
||||
}
|
||||
hts_mutexrelease(&job->lock);
|
||||
if (done || mtime_local() >= deadline)
|
||||
break;
|
||||
Sleep(poll_ms);
|
||||
if (poll_ms < 50) /* short first polls keep a fast resolve fast */
|
||||
poll_ms *= 2;
|
||||
}
|
||||
dns_job_release(job);
|
||||
return count;
|
||||
}
|
||||
|
||||
int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
|
||||
const char **error) {
|
||||
char BIGSTK host[HTS_URLMAXSIZE * 2];
|
||||
SOCaddr resolved[HTS_MAXADDRNUM];
|
||||
coucal cache;
|
||||
int count, i;
|
||||
|
||||
assertf(opt != NULL);
|
||||
assertf(_iadr != NULL);
|
||||
assertf(out != NULL);
|
||||
if (!strnotempty(iadr) || max <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
strcpybuff(iadr, jump_identification_const(_iadr));
|
||||
// couper éventuel :
|
||||
/* cache key and resolver input: identification and any ":port" stripped */
|
||||
strcpybuff(host, jump_identification_const(iadr));
|
||||
{
|
||||
char *a;
|
||||
|
||||
if ((a = jump_toport(iadr)))
|
||||
if ((a = jump_toport(host)))
|
||||
*a = '\0';
|
||||
}
|
||||
|
||||
/* get IP from the dns cache */
|
||||
count = hts_ghbn_all(cache, iadr, out, max);
|
||||
hts_mutexlock(&opt->state.lock);
|
||||
cache = hts_cache(opt);
|
||||
#if HTS_INET6 != 0
|
||||
hts_resolver_check_env(); /* settle the backend before a worker reads it */
|
||||
#endif
|
||||
count = hts_ghbn_all(cache, host, out, max);
|
||||
hts_mutexrelease(&opt->state.lock);
|
||||
if (count >= 0) { // cache hit (0 == negative-cached)
|
||||
return count;
|
||||
} else { // non présent dans le cache dns, tester
|
||||
SOCaddr resolved[HTS_MAXADDRNUM];
|
||||
t_dnscache *record;
|
||||
int i;
|
||||
}
|
||||
|
||||
#if DEBUGDNS
|
||||
printf("resolving (not cached) %s\n", iadr);
|
||||
printf("resolving (not cached) %s\n", host);
|
||||
#endif
|
||||
|
||||
count = hts_dns_resolve_nocache_list(iadr, resolved, HTS_MAXADDRNUM, error);
|
||||
/* Resolve with no lock held: getaddrinfo can block for a long time, and
|
||||
state.lock also gates the stop request (#606). */
|
||||
count = hts_dns_resolve_nocache_list_bounded(host, resolved, HTS_MAXADDRNUM,
|
||||
opt->timeout, error);
|
||||
|
||||
#if HTS_WIDE_DEBUG
|
||||
DEBUG_W("gethostbyname done\n");
|
||||
DEBUG_W("gethostbyname done\n");
|
||||
#endif
|
||||
|
||||
/* attempt to store new entry (coucal owns it and dups the host key) */
|
||||
record = malloct(sizeof(t_dnscache));
|
||||
if (count < 0) { /* timed out: no answer to cache, and none to report */
|
||||
if (error != NULL)
|
||||
*error = "host name resolution timed out";
|
||||
return 0;
|
||||
}
|
||||
|
||||
hts_mutexlock(&opt->state.lock);
|
||||
{ /* store the full list (coucal owns the record and dups the host key; a
|
||||
concurrent resolve of the same host replaces, and frees, this one) */
|
||||
t_dnscache *const record = malloct(sizeof(t_dnscache));
|
||||
|
||||
if (record != NULL) {
|
||||
memset(record, 0, sizeof(*record));
|
||||
record->host_count = count;
|
||||
@@ -5137,28 +5262,13 @@ static int hts_dns_resolve_list_(httrackp *opt, const char *_iadr,
|
||||
memcpy(record->host_addr[i], &SOCaddr_sockaddr(resolved[i]),
|
||||
record->host_length[i]);
|
||||
}
|
||||
coucal_add_pvoid(cache, iadr, record);
|
||||
coucal_add_pvoid(cache, host, record);
|
||||
}
|
||||
|
||||
/* copy result to caller (cache store may have failed; result still valid)
|
||||
*/
|
||||
for (i = 0; i < count && i < max; i++) {
|
||||
SOCaddr_copy_SOCaddr(out[i], resolved[i]);
|
||||
}
|
||||
return count;
|
||||
} // retour hp du cache
|
||||
}
|
||||
|
||||
int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
|
||||
const char **error) {
|
||||
int count;
|
||||
|
||||
if (!strnotempty(iadr) || max <= 0) {
|
||||
return 0;
|
||||
}
|
||||
hts_mutexlock(&opt->state.lock);
|
||||
count = hts_dns_resolve_list_(opt, iadr, out, max, error);
|
||||
hts_mutexrelease(&opt->state.lock);
|
||||
|
||||
/* copy result to caller (cache store may have failed; result still valid) */
|
||||
dns_copy_addrs(out, resolved, count, max);
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ typedef struct lien_adrfilsave lien_adrfilsave;
|
||||
|
||||
/* définitions globales */
|
||||
#include "htsglobal.h"
|
||||
#include "htsurlport.h"
|
||||
|
||||
/* basic net definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -222,7 +223,8 @@ LLint http_xfread1(htsblk * r, int bufl);
|
||||
/* Cached resolver: fill out[0..count-1] with up to max addresses for iadr (in
|
||||
resolver order), returning the count (0 = does not resolve, negative-cached).
|
||||
Resolves once per host; later calls read the DNS cache. Must hold no lock
|
||||
(brackets opt->state.lock itself). */
|
||||
(brackets opt->state.lock itself, never across the resolve). A miss resolves
|
||||
on a worker thread bounded by opt->timeout; a timeout reports 0, uncached. */
|
||||
int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
|
||||
const char **error);
|
||||
HTS_INLINE SOCaddr *hts_dns_resolve2(httrackp *opt, const char *iadr,
|
||||
|
||||
@@ -1532,8 +1532,30 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// last segment
|
||||
wsave[j++] = '/';
|
||||
#define MAX_UTF8_SEQ_CHARS 4
|
||||
for(i = lastSeg; wsave[i] != '\0' && j < maxLen; i++) {
|
||||
wsave[j++] = wsave[i];
|
||||
{
|
||||
// #623: the ".delayed" placeholder marker sits at the tail; cutting
|
||||
// through it drops IS_DELAYED_EXT, so the file is never renamed to its
|
||||
// final name. Reserve the trailing ".<id>.delayed" across the cut.
|
||||
size_t markStart = wsaveLen;
|
||||
if (IS_DELAYED_EXT(afs->save)) {
|
||||
const size_t extDot = wsaveLen - strlen("." DELAYED_EXT);
|
||||
size_t p = extDot; /* walk back over a dot-separated ".<hexid>" tag */
|
||||
|
||||
while (p > lastSeg && ((wsave[p - 1] >= '0' && wsave[p - 1] <= '9') ||
|
||||
(wsave[p - 1] >= 'a' && wsave[p - 1] <= 'f')))
|
||||
p--;
|
||||
// keep the tag only if truly ".<hexid>.delayed"; else the bare marker
|
||||
// (a wholly-hex base, e.g. a hashed #133 name, must not be absorbed)
|
||||
markStart = (p > lastSeg && p < extDot && wsave[p - 1] == '.')
|
||||
? p - 1
|
||||
: extDot;
|
||||
}
|
||||
// head, bounded so the marker still fits, then the marker itself
|
||||
for (i = lastSeg; i < markStart && j + (wsaveLen - markStart) < maxLen;
|
||||
i++)
|
||||
wsave[j++] = wsave[i];
|
||||
for (i = markStart; i < wsaveLen && j < maxLen; i++)
|
||||
wsave[j++] = wsave[i];
|
||||
}
|
||||
// terminating \0
|
||||
wsave[j++] = '\0';
|
||||
|
||||
@@ -315,6 +315,43 @@ static void escape_url_parens(char *const s, const size_t size) {
|
||||
strlcpybuff(s, buff, size);
|
||||
}
|
||||
|
||||
/* Strip a default ":80" from lien's authority in place. Any spelling that
|
||||
range-parses to 80 (":80", ":080") is dropped by its matched length, not a
|
||||
hardcoded 3 chars; a value that merely wraps to 80 as an int (#614) stays. */
|
||||
void hts_strip_default_port(char *lien, size_t size) {
|
||||
char *a;
|
||||
|
||||
if (!link_has_authority(lien))
|
||||
return;
|
||||
a = strstr(lien, "//"); // "//" authority
|
||||
if (a)
|
||||
a += 2;
|
||||
else
|
||||
a = lien;
|
||||
a = jump_toport(a);
|
||||
if (a) { // port present
|
||||
char *b = a + 1;
|
||||
char saved;
|
||||
int port;
|
||||
hts_boolean is_default;
|
||||
|
||||
while (isdigit((unsigned char) *b))
|
||||
b++;
|
||||
saved = *b;
|
||||
*b = '\0';
|
||||
is_default = hts_parse_url_port(a + 1, &port) && port == 80;
|
||||
*b = saved;
|
||||
if (is_default) { // default port, strip it
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
|
||||
tempo[0] = '\0';
|
||||
strncatbuff(tempo, lien, a - lien);
|
||||
strcatbuff(tempo, b); // skip the whole matched :port
|
||||
strlcpybuff(lien, tempo, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Main parser */
|
||||
int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
@@ -2138,38 +2175,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
} while((b != a) && (b));
|
||||
}
|
||||
}
|
||||
// éliminer les éventuels :80 (port par défaut!)
|
||||
if (link_has_authority(lien)) {
|
||||
char *a;
|
||||
|
||||
a = strstr(lien, "//"); // "//" authority
|
||||
if (a)
|
||||
a += 2;
|
||||
else
|
||||
a = lien;
|
||||
a = jump_toport(a);
|
||||
if (a) { // port
|
||||
int port = 0;
|
||||
int defport = 80;
|
||||
char *b = a + 1;
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
#endif
|
||||
while(isdigit((unsigned char) *b)) {
|
||||
port *= 10;
|
||||
port += (int) (*b - '0');
|
||||
b++;
|
||||
}
|
||||
if (port == defport) { // port 80, default - c'est débile
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
|
||||
tempo[0] = '\0';
|
||||
strncatbuff(tempo, lien, a - lien);
|
||||
strcatbuff(tempo, a + 3); // sauter :80
|
||||
strcpybuff(lien, tempo);
|
||||
}
|
||||
}
|
||||
}
|
||||
// drop a default :80 port from the authority
|
||||
hts_strip_default_port(lien, sizeof(lien));
|
||||
// filtrer les parazites (mailto & cie)
|
||||
/*
|
||||
if (strfield(lien,"mailto:")) { // ne pas traiter
|
||||
|
||||
@@ -106,6 +106,10 @@ struct htsmoduleStructExtended {
|
||||
*/
|
||||
int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
|
||||
|
||||
/* Strip a default ":80" (any spelling) from an absolute link's authority, in
|
||||
place into a buffer of the given size. */
|
||||
void hts_strip_default_port(char *lien, size_t size);
|
||||
|
||||
/*
|
||||
Check for 301,302.. errors ("moved") and handle them; re-isuue requests, make
|
||||
rediretc file, handle filters considerations..
|
||||
|
||||
@@ -491,14 +491,11 @@ static int socks5_handshake_stream(httrackp *opt, socks5_stream *st,
|
||||
if ((unsigned char) host[i] < ' ')
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
|
||||
}
|
||||
if (portsep != NULL) {
|
||||
int p = -1;
|
||||
|
||||
sscanf(portsep + 1, "%d", &p);
|
||||
if (p <= 0 || p > 65535)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
|
||||
port = p;
|
||||
}
|
||||
// the old range check ran after sscanf("%d") had wrapped a huge value into a
|
||||
// plausible port (#614). An empty "host:" stays refused here, unlike the
|
||||
// direct path, as it was before #614.
|
||||
if (portsep != NULL && !hts_parse_url_port(portsep + 1, &port))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
|
||||
if (link_has_authorization(proxy_name)) {
|
||||
if (!socks5_credentials(opt, proxy_name, user, sizeof(user), &userlen, pass,
|
||||
sizeof(pass), &passlen, msg, msgsize))
|
||||
|
||||