Compare commits
15 Commits
release-3.
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d0e2222d4 | ||
|
|
4166200465 | ||
|
|
2382ca3aa0 | ||
|
|
68a9a247d6 | ||
|
|
86bc02d8da | ||
|
|
e675d60301 | ||
|
|
1c8b93d9e7 | ||
|
|
d267aaf1bf | ||
|
|
226702fcab | ||
|
|
491774abda | ||
|
|
dbad05fcf0 | ||
|
|
db19089448 | ||
|
|
eb9e353f0b | ||
|
|
8a8fd7f9c0 | ||
|
|
1c1c0b1267 |
8
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Bumps builtin-baseline in src/vcpkg.json so the pinned OpenSSL/zlib can't rot.
|
||||
# windows-build.yml validates each bump; the pin makes what ships reproducible.
|
||||
- package-ecosystem: vcpkg
|
||||
directory: /src
|
||||
schedule:
|
||||
interval: weekly
|
||||
16
.github/workflows/windows-build.yml
vendored
@@ -44,6 +44,20 @@ jobs:
|
||||
shell: pwsh
|
||||
run: vcpkg integrate install
|
||||
|
||||
# The runner image's vcpkg checkout is pinned to some commit; our manifest's
|
||||
# builtin-baseline is usually newer, so `git show <baseline>:versions/...`
|
||||
# fails until that commit is local. Fetch exactly the pinned baseline (read
|
||||
# from the manifest, so Dependabot bumps need no workflow edit).
|
||||
- name: Fetch the pinned vcpkg baseline
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
$root = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { "C:\vcpkg" }
|
||||
$baseline = (Get-Content src/vcpkg.json -Raw | ConvertFrom-Json).'builtin-baseline'
|
||||
if (-not $baseline) { throw "no builtin-baseline in src/vcpkg.json" }
|
||||
git -C $root fetch --no-tags origin $baseline
|
||||
if ($LASTEXITCODE -ne 0) { throw "could not fetch vcpkg baseline $baseline" }
|
||||
|
||||
# httrack and webhttrack carry a ProjectReference to libhttrack, so building
|
||||
# them builds it first; proxytrack is standalone.
|
||||
- name: Build
|
||||
@@ -188,7 +202,7 @@ jobs:
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
expected_skips=" 48_local-crange-memresume.test" # pending #581
|
||||
expected_skips=" 48_local-crange-memresume.test 71_local-crange-repaircache.test" # pending #581
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
30
AGENTS.md
@@ -19,6 +19,13 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
(`request_queue_size`) so macOS/BSD don't drop connections under a parallel
|
||||
`-c16` bigcrawl the way Python's default backlog of 5 did.
|
||||
Or run `sh build.sh` to do bootstrap + configure + make in one shot.
|
||||
- A `tests/NN_*.test` runs only if listed in `tests/Makefile.am`'s `TESTS`; an
|
||||
unregistered file is silently skipped.
|
||||
- `make check` prepends the build's `src/` to `PATH`, but a hand-run `.test` does
|
||||
not — an installed `/usr/bin/httrack` then shadows your build. Run via `make
|
||||
check`, or `PATH="<bld>/src:$PATH"` for a manual run.
|
||||
- Give new `.test` scripts `set -e`: the older ones predate the rule, so several
|
||||
`local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
|
||||
|
||||
## Hard invariants
|
||||
- **Generated autotools files are NOT in git.** `configure`, every
|
||||
@@ -31,15 +38,32 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
- **Format only changed lines** with `git clang-format` (clang-format 19). Never
|
||||
reformat untouched code: the engine was formatted by an old tool and won't
|
||||
round-trip.
|
||||
- **Byte-safe edits.** Files with raw high bytes are ISO-8859-1 (French
|
||||
comments). Edit them byte-wise (`perl -0pi`, `sed`), not through a tool that
|
||||
re-encodes to UTF-8 and corrupts them.
|
||||
- **Byte-safe edits.** A few tracked files carry raw ISO-8859-1 high bytes
|
||||
(French comments): `src/htsconcat.c`, `lang/*.txt`, `html/contact.html`, and
|
||||
the `fuzz/corpus/*` vectors. Edit those byte-wise (`perl -0pi`, `sed`), not
|
||||
through a tool that re-encodes to UTF-8 and corrupts them. The rest of the tree
|
||||
is UTF-8 and safe to edit normally.
|
||||
|
||||
## Security (HTTrack parses hostile input off the network)
|
||||
- Bounds-check every copy. Overflow-safe form: put the untrusted value alone,
|
||||
`untrusted < limit - controlled` — never `controlled + untrusted < limit`,
|
||||
which can wrap and pass.
|
||||
|
||||
## C conventions
|
||||
- **Use the `*t` allocator wrappers, never raw libc** (`htssafe.h`):
|
||||
`malloct`/`calloct`/`realloct`/`freet`/`strdupt`, in test and selftest code
|
||||
too. `freet` NULLs its (lvalue) argument and tolerates NULL; `calloct(n, sz)`
|
||||
keeps calloc's arg order. Only exception: storing or calling a libc symbol
|
||||
itself (e.g. a resolver-backend function pointer).
|
||||
- **Exported API is `HTSEXT_API`.** Everything else is hidden by
|
||||
`-fvisibility=hidden` and free to change (check with `nm -D --defined-only
|
||||
libhttrack.so`). Touching an installed-header struct (see `DevIncludes_DATA` in
|
||||
`src/Makefile.am`) or an exported signature is an ABI break — flag and discuss,
|
||||
bump the soname, and prefer keeping the old entry point beside a new one.
|
||||
- **Windows ABI is free to break, POSIX is not.** The Windows DLL ships next to
|
||||
the exe with no soname contract, so a `_WIN32`-only ABI change needs no
|
||||
deprecation dance; POSIX/ELF keeps the flag-discuss-bump rules.
|
||||
|
||||
## Code & prose
|
||||
- Be terse. Comment the why, in English; translate French comments you touch.
|
||||
- Strip AI tells from prose (em-dash overuse, rule-of-three, filler, vague
|
||||
|
||||
@@ -579,7 +579,7 @@ And then, put the email address in your pages through:
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
223
html/android.html
Normal file
@@ -0,0 +1,223 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="How to mirror a website with HTTrack on Android: install the app, create a project, enter the address, run the mirror, and browse the result on your device." />
|
||||
<meta name="keywords" content="httrack, HTTrack, android, offline browser, web mirror utility, website mirroring, mobile, Google Play" />
|
||||
<title>HTTrack on Android</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>HTTrack on Android</em></h2>
|
||||
|
||||
<br>
|
||||
|
||||
<p>HTTrack downloads a website to your device so you can read it offline. The
|
||||
Android app runs the same mirroring engine as the desktop version behind a touch
|
||||
interface. It needs Android 7.0 or later, and is available on
|
||||
<a href="https://play.google.com/store/apps/details?id=com.httrack.android">Google Play</a>.</p>
|
||||
|
||||
<p>The steps below follow one mirror from start to finish.</p>
|
||||
|
||||
<br>
|
||||
|
||||
<ol>
|
||||
|
||||
<li><b>Grant storage access</b></li>
|
||||
<br><small>On first launch the app asks for permission to store mirrors on your
|
||||
device. The prompt reads <em>"Allow HTTrack Website Copier to access photos,
|
||||
media, and files on your device?"</em>. Tap <b>ALLOW</b>: without it the app
|
||||
cannot save the downloaded files.</small>
|
||||
<br><br><center><img src="img/android_permission.png" width="320" alt="First-run storage permission dialog" border="0"></center>
|
||||
<br><small>If you have used an older release, a second prompt offers to bring its
|
||||
mirrors into the app. Tap <b>Import</b> to move them, or <b>Not now</b> to
|
||||
skip.</small>
|
||||
<br><br><center><img src="img/android_import.png" width="320" alt="Import mirrors from an older version" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Start</b></li>
|
||||
<br><small>The welcome screen shows the engine version at the bottom. Tap
|
||||
<b>Next</b> to create a project, or <b>Browse sites</b> to open a mirror you
|
||||
already made.</small>
|
||||
<br><br><center><img src="img/android_startup.png" width="320" alt="Welcome screen" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Name the project</b></li>
|
||||
<br><small>Give the project a name, and optionally a category to group related
|
||||
mirrors. <b>Base path</b> shows where the files will be written. Tap
|
||||
<b>Next</b>.</small>
|
||||
<br><br><center><img src="img/android_project.png" width="320" alt="Project name, category, and base path" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Enter the address</b></li>
|
||||
<br><small>Type the site address. If a project of the same name already exists,
|
||||
pick <b>Continue interrupted download</b> or <b>Update existing download</b>.
|
||||
Tap <b>Options</b> to adjust the crawl, or <b>Start</b> to begin.</small>
|
||||
<br><br><center><img src="img/android_url.png" width="320" alt="Web address and download action" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Options (optional)</b></li>
|
||||
<br><small>The <b>Options</b> screen holds the crawl settings, grouped into
|
||||
eleven tabs. It uses the same profile format as the desktop version, so a tab you
|
||||
know from WinHTTrack behaves the same way here.</small>
|
||||
<br><br><center><img src="img/android_options.png" width="320" alt="Options tab list" border="0"></center>
|
||||
<br><small>The tabs are:</small>
|
||||
<ul>
|
||||
<li><small><b>Scan Rules</b>: wildcard filters for the addresses to keep or skip.</small></li>
|
||||
<li><small><b>Limits</b>: caps on depth, size, time, speed, and number of connections.</small></li>
|
||||
<li><small><b>Flow Control</b>: simultaneous connections, timeouts, and retries.</small></li>
|
||||
<li><small><b>Links</b>: how far to follow links, and which extra files to fetch.</small></li>
|
||||
<li><small><b>Build</b>: how the saved files and folders are named.</small></li>
|
||||
<li><small><b>Browser ID</b>: the user-agent, language, and headers sent to servers.</small></li>
|
||||
<li><small><b>Spider</b>: cookies, robots.txt handling, and request behavior.</small></li>
|
||||
<li><small><b>Proxy address</b>: proxy host and port.</small></li>
|
||||
<li><small><b>Log, Index, Cache</b>: log files, the search index, and the update cache.</small></li>
|
||||
<li><small><b>Type/MIME associations</b>: map file extensions to MIME types.</small></li>
|
||||
<li><small><b>Experts Only</b>: scan mode, how far the crawl may travel, and link rewriting.</small></li>
|
||||
</ul>
|
||||
<br><center><img src="img/android_scanrules.png" width="320" alt="Scan Rules tab" border="0"></center>
|
||||
<br><center><img src="img/android_experts.png" width="320" alt="Experts Only tab" border="0"></center>
|
||||
<br><small>The <a href="step9.html">desktop option reference</a> describes every
|
||||
setting in full. A few desktop options are missing or fixed on Android: the proxy
|
||||
tab has a host and port but no login and password, the download folder is fixed
|
||||
inside the app's storage so there is no output-path setting, and the MIME table
|
||||
holds up to eight entries.</small>
|
||||
<br><br>
|
||||
|
||||
<li><b>Run the mirror</b></li>
|
||||
<br><small>HTTrack downloads the site and reports live figures: bytes saved,
|
||||
links scanned, transfer rate, and errors. Tap <b>Abort</b> to stop early; a
|
||||
partial mirror can be resumed later.</small>
|
||||
<br><br><center><img src="img/android_progress.png" width="320" alt="Crawl in progress with live statistics" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Browse the result</b></li>
|
||||
<br><small>When the crawl finishes the app reports <b>Success</b> and the mirror
|
||||
location. Tap <b>Browse Mirrored Website</b> to read the copy in your browser,
|
||||
<b>View log</b> to see what happened, or <b>New project</b> to start
|
||||
again.</small>
|
||||
<br><br><center><img src="img/android_finished.png" width="320" alt="Mirror finished, success" border="0"></center>
|
||||
<br><br>
|
||||
|
||||
<li><b>Where the files are</b></li>
|
||||
<br><small>Mirrors are written under <tt>/storage/emulated/0/HTTrack/Websites</tt>,
|
||||
in a folder named after the project. That folder lives in your device's shared
|
||||
storage, so a file manager or a USB connection can reach it. Opening
|
||||
<tt>index.html</tt> there browses a mirror without the app.</small>
|
||||
|
||||
</ol>
|
||||
|
||||
<br><br><br><br>
|
||||
<p align="right">Back to <a href="index.html">Home</a></p>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -173,7 +173,7 @@ There are also specific issues regarding this format:
|
||||
<ul>
|
||||
<li>The data in the central directory (such as CD extra field, and CD comments) are not used</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^16 files (65535) ; in such case the total number of entries in the 32-bit central directory is 65536 (0xffff), but the presence of the 64-bit central directory is not mandatory</li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory must be present <b>(not currently supported)</b></li>
|
||||
<li>The ZIP archive is allowed to contains more than 2^32 bytes (4GiB) ; in such case the 64-bit central directory is emitted automatically (a single stored entry of 4GiB or more is not supported)</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
@@ -282,7 +282,7 @@ Libraries should generally handle this peculiar format, however.
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ The command-line version
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -243,7 +243,7 @@ roche at httrack dot com (Xavier ROCHE)<br>
|
||||
<br><hr><br>
|
||||
<br>
|
||||
This program is covered by the GNU General Public License.<br>
|
||||
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2007 Xavier Roche and other contributors
|
||||
HTTrack/HTTrack Website Copier is Copyright (C) 1998-2026 Xavier Roche and other contributors
|
||||
<br>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
@@ -259,7 +259,7 @@ roche at httrack dot com (Xavier ROCHE)<br>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -147,7 +147,7 @@ This page describes the HTTrack cache format.
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -263,7 +263,7 @@ clear language!
|
||||
|
||||
<li><a href="#QM8">Can HTTrack generates HP-UX or ISO9660 compatible files?</a><br></li>
|
||||
|
||||
<li><a href="#QM9">If there any SOCKS support?</a><br></li>
|
||||
<li><a href="#QM9">Is there any SOCKS support?</a><br></li>
|
||||
|
||||
<li><a href="#QM10">What's this hts-cache directory? Can I remove it?</a><br></li>
|
||||
|
||||
@@ -835,8 +835,8 @@ A: <em>Yes. Use user:password@your_proxy_name as your proxy name (example: <tt>s
|
||||
<br><br><a NAME="QM8">Q: <strong>Can HTTrack generates HP-UX or ISO9660 compatible files?</strong></a><br>
|
||||
A: <em>Yes. See the build options (-N, or see the WinHTTrack options)</em>
|
||||
|
||||
<br><br><a NAME="QM9">Q: <strong>If there any SOCKS support?</strong></a><br>
|
||||
A: <em>Not yet!</em>
|
||||
<br><br><a NAME="QM9">Q: <strong>Is there any SOCKS support?</strong></a><br>
|
||||
A: <em>Yes. HTTrack supports SOCKS5 and HTTP CONNECT proxies: give the proxy with a scheme prefix, e.g. <tt>-P socks5://host:port</tt> or <tt>-P connect://host:port</tt> (prefix <tt>user:pass@</tt> before the host for authenticated proxies).</em>
|
||||
|
||||
<br><br><a NAME="QM10">Q: <strong>What's this hts-cache directory? Can I remove it?</strong></a><br>
|
||||
A: <em>NO if you want to update the site, because this directory is used by HTTrack for this purpose.
|
||||
@@ -931,7 +931,7 @@ A: <em>Feel free to <a href="contact.html">contact us</a>!
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -189,7 +189,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<br>
|
||||
Filters are analyzed by HTTrack from the first filter to the last one. The complete URL
|
||||
name is compared to filters defined by the user or added automatically by HTTrack. <br><br>
|
||||
A scan rule has an higher priority is it is declared later - hierarchy is important: <br>
|
||||
A scan rule has a higher priority if it is declared later - hierarchy is important: <br>
|
||||
|
||||
<br>
|
||||
<table BORDER="1" CELLPADDING="2">
|
||||
@@ -307,7 +307,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<br>
|
||||
Filters are analyzed by HTTrack from the first filter to the last one. The sizes
|
||||
are compared against scan rules defined by the user.<br><br>
|
||||
A scan rule has an higher priority is it is declared later - hierarchy is important.<br>
|
||||
A scan rule has a higher priority if it is declared later - hierarchy is important.<br>
|
||||
|
||||
Note: scan rules based on size can be mixed with regular URL patterns<br>
|
||||
|
||||
@@ -361,7 +361,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<br>
|
||||
Filters are analyzed by HTTrack from the first filter to the last one. The complete MIME
|
||||
type is compared against scan rules defined by the user.<br><br>
|
||||
A scan rule has an higher priority is it is declared later - hierarchy is important<br>
|
||||
A scan rule has a higher priority if it is declared later - hierarchy is important<br>
|
||||
|
||||
Note: scan rules based on MIME types can <b>NOT</b> be mixed with regular URL patterns or size patterns within the same rule, but you can use both of them in distinct ones<br>
|
||||
|
||||
@@ -387,12 +387,12 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<tr>
|
||||
<td nowrap><tt>-mime:video/*</tt></td>
|
||||
<td>This will refuse all video links that were already scheduled for download
|
||||
(i.e. all other 'application/' link download will be aborted)</td>
|
||||
(i.e. the download will be aborted)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><tt>-mime:video/* -mime:audio/*</tt></td>
|
||||
<td>This will refuse all audio and video links that were already scheduled for download
|
||||
(i.e. all other 'application/' link download will be aborted)</td>
|
||||
(i.e. the download will be aborted)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><tt>-mime:*/* +mime:text/html +mime:image/*</tt></td>
|
||||
@@ -466,7 +466,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
BIN
html/img/android_experts.png
Normal file
|
After Width: | Height: | Size: 94 KiB |
BIN
html/img/android_finished.png
Normal file
|
After Width: | Height: | Size: 32 KiB |
BIN
html/img/android_import.png
Normal file
|
After Width: | Height: | Size: 50 KiB |
BIN
html/img/android_options.png
Normal file
|
After Width: | Height: | Size: 33 KiB |
BIN
html/img/android_permission.png
Normal file
|
After Width: | Height: | Size: 48 KiB |
BIN
html/img/android_progress.png
Normal file
|
After Width: | Height: | Size: 33 KiB |
BIN
html/img/android_project.png
Normal file
|
After Width: | Height: | Size: 30 KiB |
BIN
html/img/android_scanrules.png
Normal file
|
After Width: | Height: | Size: 41 KiB |
BIN
html/img/android_startup.png
Normal file
|
After Width: | Height: | Size: 36 KiB |
BIN
html/img/android_url.png
Normal file
|
After Width: | Height: | Size: 33 KiB |
@@ -111,6 +111,12 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li><a href="shelldoc.html">WinHTTrack/WebHTTrack (GUI version for Windows or Linux/Unix)</a></li>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix relese (WebHTTrack)<br>
|
||||
<br>
|
||||
<li><a href="android.html">HTTrack on Android</a></li>
|
||||
<br>Step-by-step guide for the Android app<br>
|
||||
<br>
|
||||
<li><a href="cmddoc.html">Command-line version</a></li>
|
||||
<br>How to run HTTrack from the command line, and where to find every option<br>
|
||||
<br>
|
||||
<li><a href="fcguide.html">HTTrack Users Guide By Fred Cohen</a></li>
|
||||
<br>A tutorial that describes all command-line options, for Linux and Windows users<br>
|
||||
<br>
|
||||
@@ -142,7 +148,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ You may also want to check the <tt>httrack.c</tt> and <tt>httrack.h<tt> files to
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -105,238 +105,14 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li>Filters: <a href="filters.html">how to use them</a></li>
|
||||
<br><small>Here you can find informations on filters: how to accept all gif files in a mirror, for example</small>
|
||||
<br><br>
|
||||
<li>List of options</li>
|
||||
</ul>
|
||||
</ul>
|
||||
|
||||
<tt>
|
||||
<pre>
|
||||
<p>The complete, current list of command-line options is generated from the
|
||||
program itself, in <a href="httrack.man.html">the httrack manual page</a>.</p>
|
||||
|
||||
w mirror with automatic wizard
|
||||
This is the default scanning option, the engine automatically scans links according to the default options, and filters defined. It does not prompt a message when a "foreign" link is reached.
|
||||
<p>WinHTTrack and WebHTTrack offer the same options through their graphical option
|
||||
panels, described in the <a href="step9.html">step-by-step guide</a>.</p>
|
||||
|
||||
W semi-automatic mirror with help-wizard (asks questions)
|
||||
This option lets the engine ask the user if a link must be mirrored or not, when a new web has been found.
|
||||
|
||||
g just get files (saved in the current directory)
|
||||
This option forces the engine not to scan the files indicated - i.e. the engine only gets the files indicated.
|
||||
|
||||
i continue an interrupted mirror using the cache
|
||||
This option indicates to the engine that a mirror must be updated or continued.
|
||||
|
||||
rN recurse get with limited link depth of N
|
||||
This option sets the maximum recurse level. Default is infinite (the engine "knows" that it should not go out of current domain)
|
||||
|
||||
a stay on the same address
|
||||
This is the default primary scanning option, the engine does not go out of domains without permissions (filters, for example)
|
||||
|
||||
d stay on the same principal domain
|
||||
This option lets the engine go on all sites that exist on the same principal domain.
|
||||
Example: a link located at www.example.com that goes to members.example.com will be followed.
|
||||
|
||||
l stay on the same location (.com, etc.)
|
||||
This option lets the engine go on all sites that exist on the same location.
|
||||
Example: a link located at www.example.com that goes to www.anyotherweb.com will be followed.
|
||||
Warning: this is a potentially dangerous option, limit the recurse depth with r option.
|
||||
|
||||
e go everywhere on the web
|
||||
This option lets the engine go on any sites.
|
||||
Example: a link located at www.example.com that goes to www.anyotherweb.org will be followed.
|
||||
Warning: this is a potentially dangerous option, limit the recurse depth with r option.
|
||||
|
||||
n get non-html files 'near' an html file (ex: an image located outside)
|
||||
This option lets the engine catch all files that have references on a page, but that exist outside the web site.
|
||||
Example: List of ZIP files links on a page.
|
||||
|
||||
t test all URLs (even forbidden ones)
|
||||
This option lets the engine test all links that are not caught.
|
||||
Example: to test broken links in a site
|
||||
|
||||
x replace external html links by error pages
|
||||
This option tells the engine to rewrite all links not taken into warning pages.
|
||||
Example: to browse offline a site, and to warn people that they must be online if they click to external links.
|
||||
|
||||
sN follow robots.txt and meta robots tags
|
||||
This option sets the way the engine treats "robots.txt" files. This file is often set by webmasters to avoir cgi-bin directories, or other irrevelant pages.
|
||||
Values:
|
||||
s0 Do not take robots.txt rules
|
||||
s1 Follow rules, if compatible with internal filters
|
||||
s2 Always follow site's rules
|
||||
|
||||
bN accept cookies in cookies.txt
|
||||
This option activates or unactivates the cookie
|
||||
b0 do not accept cookies
|
||||
b1 accept cookies
|
||||
|
||||
S stay on the same directory
|
||||
This option asks the engine to stay on the same folder level.
|
||||
Example: A link in /index.html that points to /sub/other.html will not be followed
|
||||
|
||||
D can only go down into subdirs
|
||||
This is the default option, the engine can go everywhere on the same directoy, or in lower structures
|
||||
|
||||
U can only go to upper directories
|
||||
This option asks the engine to stay on the same folder level or in upper structures
|
||||
|
||||
B can both go up&down into the directory structure
|
||||
This option lets the engine to go in any directory level
|
||||
|
||||
Y mirror ALL links located in the first level pages (mirror links)
|
||||
This option is activated for the links typed in the command line
|
||||
Example: if you have a list of web sites in www.asitelist.com/index.html, then all these sites will be mirrored
|
||||
|
||||
NN name conversion type (0 *original structure 1,2,3 html/data in one directory)
|
||||
N0 Site-structure (default)
|
||||
N1 Html in web/, images/other files in web/images/
|
||||
N2 Html in web/html, images/other in web/images
|
||||
N3 Html in web/, images/other in web/
|
||||
N4 Html in web/, images/other in web/xxx, where xxx is the file extension (all gif will be placed onto web/gif, for example)
|
||||
N5 Images/other in web/xxx and Html in web/html
|
||||
|
||||
N99 All files in web/, with random names (gadget !)
|
||||
|
||||
N100 Site-structure, without www.domain.xxx/
|
||||
N101 Identical to N1 except that "web" is replaced by the site's name
|
||||
N102 Identical to N2 except that "web" is replaced by the site's name
|
||||
N103 Identical to N3 except that "web" is replaced by the site's name
|
||||
N104 Identical to N4 except that "web" is replaced by the site's name
|
||||
N105 Identical to N5 except that "web" is replaced by the site's name
|
||||
N199 Identical to N99 except that "web" is replaced by the site's name
|
||||
|
||||
N1001 Identical to N1 except that there is no "web" directory
|
||||
N1002 Identical to N2 except that there is no "web" directory
|
||||
N1003 Identical to N3 except that there is no "web" directory (option set for g option)
|
||||
N1004 Identical to N4 except that there is no "web" directory
|
||||
N1005 Identical to N5 except that there is no "web" directory
|
||||
N1099 Identical to N99 except that there is no "web" directory
|
||||
|
||||
LN long names
|
||||
L0 Filenames and directory names are limited to 8 characters + 3 for extension
|
||||
L1 No restrictions (default)
|
||||
|
||||
K keep original links (e.g. http://www.adr/link) (K0 *relative link)
|
||||
This option has only been kept for compatibility reasons
|
||||
|
||||
pN priority mode:
|
||||
p0 just scan, don't save anything (for checking links)
|
||||
p1 save only html files
|
||||
p2 save only non html files
|
||||
p3 save all files
|
||||
p7 get html files before, then treat other files
|
||||
|
||||
cN number of multiple connections (*c8)
|
||||
Set the numer of multiple simultaneous connections
|
||||
|
||||
O path for mirror/logfiles+cache (-O path_mirror[,path_cache_and_logfiles])
|
||||
This option define the path for mirror and log files
|
||||
Example: -P "/user/webs","/user/logs"
|
||||
|
||||
P proxy use (-P proxy:port or -P user:pass@proxy:port)
|
||||
This option define the proxy used in this mirror
|
||||
Example: -P proxy.myhost.com:8080
|
||||
|
||||
F user-agent field (-F \"user-agent name\
|
||||
This option define the user-agent field
|
||||
Example: -F "Mozilla/4.5 (compatible; HTTrack 1.2x; Windows 98)"
|
||||
|
||||
mN maximum file length for a non-html file
|
||||
This option define the maximum size for non-html files
|
||||
Example: -m100000
|
||||
|
||||
mN,N' for non html (N) and html (N')
|
||||
This option define the maximum size for non-html files and html-files
|
||||
Example: -m100000,250000
|
||||
|
||||
MN maximum overall size that can be uploaded/scanned
|
||||
This option define the maximum amount of bytes that can be downloaded
|
||||
Example: -M1000000
|
||||
|
||||
EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)
|
||||
This option define the maximum time that the mirror can last
|
||||
Example: -E3600
|
||||
|
||||
AN maximum transfer rate in bytes/seconds (1000=1kb/s max)
|
||||
This option define the maximum transfer rate
|
||||
Example: -A2000
|
||||
|
||||
GN pause transfer if N bytes reached, and wait until lock file is deleted
|
||||
This option asks the engine to pause every time N bytes have been transferred, and restarts when the lock file "hts-pause.lock" is being deleted
|
||||
Example: -G20000000
|
||||
|
||||
u check document type if unknown (cgi,asp..)
|
||||
This option define the way the engine checks the file type
|
||||
u0 do not check
|
||||
u1 check but /
|
||||
u2 check always
|
||||
|
||||
RN number of retries, in case of timeout or non-fatal errors (*R0)
|
||||
This option sets the maximum number of tries that can be processed for a file
|
||||
|
||||
o *generate output html file in case of error (404..) (o0 don't generate)
|
||||
This option define whether the engine has to generate html output file or not if an error occurred
|
||||
|
||||
TN timeout, number of seconds after a non-responding link is shutdown
|
||||
This option define the timeout
|
||||
Example: -T120
|
||||
|
||||
JN traffic jam control, minimum transfert rate (bytes/seconds) tolerated for a link
|
||||
This option define the minimum transfer rate
|
||||
Example: -J200
|
||||
|
||||
HN host is abandoned if: 0=never, 1=timeout, 2=slow, 3=timeout or slow
|
||||
This option define whether the engine has to abandon a host if a timeout/"too slow" error occurred
|
||||
|
||||
&P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
|
||||
This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
|
||||
|
||||
j *parse scripts (j0 don't parse)
|
||||
This option defines whether the engine has to parse scripts or not to catch included files
|
||||
|
||||
I *make an index (I0 don't make)
|
||||
This option define whether the engine has to generate an index.html on the top directory
|
||||
|
||||
X *delete old files after update (X0 keep delete)
|
||||
This option define whether the engine has to delete locally, after an update, files that have been deleted in the remote mirror, or that have been excluded
|
||||
|
||||
C *create/use a cache for updates and retries (C0 no cache)
|
||||
This option define whether the engine has to generate a cache for retries and updates or not
|
||||
|
||||
k store all files in cache (not useful if files on disk)
|
||||
This option define whether the engine has to store all files in cache or not
|
||||
|
||||
V execute system command after each files ($0 is the filename: -V \"rm \\$0\
|
||||
This option lets the engine execute a command for each file saved on disk
|
||||
|
||||
q quiet mode (no questions)
|
||||
Do not ask questions (for example, for confirm an option)
|
||||
|
||||
Q log quiet mode (no log)
|
||||
Do not generate log files
|
||||
|
||||
v verbose screen mode
|
||||
Log files are printed in the screen
|
||||
|
||||
f *log file mode
|
||||
Log files are generated into two log files
|
||||
|
||||
z extra infos log
|
||||
Add more informations on log files
|
||||
|
||||
Z debug log
|
||||
Add debug informations on log files
|
||||
|
||||
|
||||
--mirror <URLs> *make a mirror of site(s)
|
||||
--get <URLs> get the files indicated, do not seek other URLs
|
||||
--mirrorlinks <URLs> test links in pages (identical to -Y)
|
||||
--testlinks <URLs> test links in pages
|
||||
--spider <URLs> spider site(s), to test links (reports Errors & Warnings)
|
||||
--update <URLs> update a mirror, without confirmation
|
||||
--skeleton<URLs> make a mirror, but gets only html files
|
||||
|
||||
--http10 force http/1.0 requests when possible
|
||||
|
||||
</pre>
|
||||
</tt>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
|
||||
@@ -352,7 +128,7 @@ Add debug informations on log files
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ downloads. HTTrack is fully configurable, and has an integrated help system.
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -286,7 +286,7 @@ module exit point
|
||||
the function name and prototype MUST match this prototype
|
||||
*/
|
||||
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
|
||||
fprintf(stder, "Module unplugged");
|
||||
fprintf(stderr, "Module unplugged");
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
@@ -331,7 +331,7 @@ httrack --wrapper mylibrary,myparameter-string http://www.example.com
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -204,7 +204,7 @@ Below additional function names that can be defined inside the optional libhttra
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -250,7 +250,7 @@ Script example:
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ ${LANG_K3} : ${HTTRACK_WEB}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -159,7 +159,7 @@ ${do:end-if}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ ${error}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ ${do:loadhash}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -143,7 +143,7 @@ ${path}/${projname}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ ${do:end-if}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ ${LANG_THANKYOU}!
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -158,7 +158,7 @@ ${do:end-if}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ ${LANG_IOPT10}:
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -250,7 +250,7 @@ ${LANG_W3}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@ ${listid:build:LISTDEF_3}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -140,7 +140,7 @@ ${do:output-mode:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -191,7 +191,7 @@ ${listid:travel3:LISTDEF_11}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -184,7 +184,7 @@ ${LANG_I46}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -228,7 +228,7 @@ ${LANG_I64b}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -156,7 +156,7 @@ ${LANG_I43b}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -148,7 +148,7 @@ ${LANG_B13}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -212,7 +212,7 @@ ${LANG_STRIPQUERY}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -166,7 +166,7 @@ ${listid:logtype:LISTDEF_9}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -204,7 +204,7 @@ ${LANG_H20} ${info.currentjob}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -283,7 +283,7 @@ ${do:end-if:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -196,7 +196,7 @@ ${do:output-mode:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -324,7 +324,7 @@ ${do:output-mode:}
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
<td id="footer"><small><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ You may encounter minor differences (in the display, or in various options) betw
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -128,7 +128,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -143,7 +143,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -157,7 +157,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -128,7 +128,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -150,7 +150,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ In this case, HTTrack won't check the type, because it has learned that "foo" is
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -181,7 +181,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -165,7 +165,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -162,7 +162,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -156,7 +156,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 2007 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
@@ -373,7 +373,8 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
|
||||
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset) {
|
||||
const UINT cp = hts_getCodepage(charset);
|
||||
|
||||
return hts_convertStringCPToUTF8(s, size, cp);
|
||||
@@ -554,7 +555,8 @@ static char *hts_convertStringCharset(const char *s, size_t size,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
|
||||
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset) {
|
||||
/* Empty string ? */
|
||||
if (size == 0) {
|
||||
return strdup("");
|
||||
|
||||
@@ -51,8 +51,8 @@ typedef unsigned int hts_UCS4;
|
||||
* Convert the string "s" from charset "charset" to UTF-8.
|
||||
* Return NULL upon error.
|
||||
**/
|
||||
extern char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset);
|
||||
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
|
||||
const char *charset);
|
||||
|
||||
/**
|
||||
* Convert the string "s" from UTF-8 to charset "charset".
|
||||
|
||||
@@ -2368,30 +2368,28 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
opt->errlog = stderr;
|
||||
} else if (httrack_logmode >= 2) {
|
||||
// deux fichiers log
|
||||
structcheck(StringBuff(opt->path_log));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"));
|
||||
// path_log holds UTF-8 bytes (argv is UTF-8): the ANSI file calls would
|
||||
// read them as the codepage and drop the logs into a mangled twin (#630).
|
||||
structcheck_utf8(StringBuff(opt->path_log));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt"));
|
||||
|
||||
/* Check FS directory structure created */
|
||||
structcheck(StringBuff(opt->path_log));
|
||||
structcheck_utf8(StringBuff(opt->path_log));
|
||||
|
||||
opt->log =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"),
|
||||
"w");
|
||||
opt->log = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-log.txt"),
|
||||
"w");
|
||||
if (httrack_logmode == 2)
|
||||
opt->errlog =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"),
|
||||
"w");
|
||||
opt->errlog = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-err.txt"),
|
||||
"w");
|
||||
else
|
||||
opt->errlog = opt->log;
|
||||
if (opt->log == NULL) {
|
||||
|
||||
@@ -1532,8 +1532,30 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// last segment
|
||||
wsave[j++] = '/';
|
||||
#define MAX_UTF8_SEQ_CHARS 4
|
||||
for(i = lastSeg; wsave[i] != '\0' && j < maxLen; i++) {
|
||||
wsave[j++] = wsave[i];
|
||||
{
|
||||
// #623: the ".delayed" placeholder marker sits at the tail; cutting
|
||||
// through it drops IS_DELAYED_EXT, so the file is never renamed to its
|
||||
// final name. Reserve the trailing ".<id>.delayed" across the cut.
|
||||
size_t markStart = wsaveLen;
|
||||
if (IS_DELAYED_EXT(afs->save)) {
|
||||
const size_t extDot = wsaveLen - strlen("." DELAYED_EXT);
|
||||
size_t p = extDot; /* walk back over a dot-separated ".<hexid>" tag */
|
||||
|
||||
while (p > lastSeg && ((wsave[p - 1] >= '0' && wsave[p - 1] <= '9') ||
|
||||
(wsave[p - 1] >= 'a' && wsave[p - 1] <= 'f')))
|
||||
p--;
|
||||
// keep the tag only if truly ".<hexid>.delayed"; else the bare marker
|
||||
// (a wholly-hex base, e.g. a hashed #133 name, must not be absorbed)
|
||||
markStart = (p > lastSeg && p < extDot && wsave[p - 1] == '.')
|
||||
? p - 1
|
||||
: extDot;
|
||||
}
|
||||
// head, bounded so the marker still fits, then the marker itself
|
||||
for (i = lastSeg; i < markStart && j + (wsaveLen - markStart) < maxLen;
|
||||
i++)
|
||||
wsave[j++] = wsave[i];
|
||||
for (i = markStart; i < wsaveLen && j < maxLen; i++)
|
||||
wsave[j++] = wsave[i];
|
||||
}
|
||||
// terminating \0
|
||||
wsave[j++] = '\0';
|
||||
|
||||
@@ -315,6 +315,43 @@ static void escape_url_parens(char *const s, const size_t size) {
|
||||
strlcpybuff(s, buff, size);
|
||||
}
|
||||
|
||||
/* Strip a default ":80" from lien's authority in place. Any spelling that
|
||||
range-parses to 80 (":80", ":080") is dropped by its matched length, not a
|
||||
hardcoded 3 chars; a value that merely wraps to 80 as an int (#614) stays. */
|
||||
void hts_strip_default_port(char *lien, size_t size) {
|
||||
char *a;
|
||||
|
||||
if (!link_has_authority(lien))
|
||||
return;
|
||||
a = strstr(lien, "//"); // "//" authority
|
||||
if (a)
|
||||
a += 2;
|
||||
else
|
||||
a = lien;
|
||||
a = jump_toport(a);
|
||||
if (a) { // port present
|
||||
char *b = a + 1;
|
||||
char saved;
|
||||
int port;
|
||||
hts_boolean is_default;
|
||||
|
||||
while (isdigit((unsigned char) *b))
|
||||
b++;
|
||||
saved = *b;
|
||||
*b = '\0';
|
||||
is_default = hts_parse_url_port(a + 1, &port) && port == 80;
|
||||
*b = saved;
|
||||
if (is_default) { // default port, strip it
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
|
||||
tempo[0] = '\0';
|
||||
strncatbuff(tempo, lien, a - lien);
|
||||
strcatbuff(tempo, b); // skip the whole matched :port
|
||||
strlcpybuff(lien, tempo, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Main parser */
|
||||
int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
@@ -2138,38 +2175,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
} while((b != a) && (b));
|
||||
}
|
||||
}
|
||||
// éliminer les éventuels :80 (port par défaut!)
|
||||
if (link_has_authority(lien)) {
|
||||
char *a;
|
||||
|
||||
a = strstr(lien, "//"); // "//" authority
|
||||
if (a)
|
||||
a += 2;
|
||||
else
|
||||
a = lien;
|
||||
a = jump_toport(a);
|
||||
if (a) { // port
|
||||
int port = 0;
|
||||
int defport = 80;
|
||||
char *b = a + 1;
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
#endif
|
||||
while(isdigit((unsigned char) *b)) {
|
||||
port *= 10;
|
||||
port += (int) (*b - '0');
|
||||
b++;
|
||||
}
|
||||
if (port == defport) { // port 80, default - c'est débile
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
|
||||
tempo[0] = '\0';
|
||||
strncatbuff(tempo, lien, a - lien);
|
||||
strcatbuff(tempo, a + 3); // sauter :80
|
||||
strcpybuff(lien, tempo);
|
||||
}
|
||||
}
|
||||
}
|
||||
// drop a default :80 port from the authority
|
||||
hts_strip_default_port(lien, sizeof(lien));
|
||||
// filtrer les parazites (mailto & cie)
|
||||
/*
|
||||
if (strfield(lien,"mailto:")) { // ne pas traiter
|
||||
|
||||
@@ -106,6 +106,10 @@ struct htsmoduleStructExtended {
|
||||
*/
|
||||
int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
|
||||
|
||||
/* Strip a default ":80" (any spelling) from an absolute link's authority, in
|
||||
place into a buffer of the given size. */
|
||||
void hts_strip_default_port(char *lien, size_t size);
|
||||
|
||||
/*
|
||||
Check for 301,302.. errors ("moved") and handle them; re-isuue requests, make
|
||||
rediretc file, handle filters considerations..
|
||||
|
||||
@@ -1589,6 +1589,41 @@ static int st_identabs(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Default-port strip (#627): a genuine 80 (any spelling) is removed by its
|
||||
matched length, host preserved; a non-80 port or one that only wraps to 80 as
|
||||
a 32-bit int (#614) is left intact. Guards the old bug where ":080"/":0080"
|
||||
dropped a hardcoded 3 chars and glued the leftover digits onto the host. */
|
||||
static int st_stripport(httrackp *opt, int argc, char **argv) {
|
||||
static const struct {
|
||||
const char *in, *out;
|
||||
} cases[] = {
|
||||
{"http://127.0.0.1:80/x", "http://127.0.0.1/x"},
|
||||
{"http://127.0.0.1:080/x", "http://127.0.0.1/x"},
|
||||
{"http://127.0.0.1:0080/x", "http://127.0.0.1/x"},
|
||||
{"http://127.0.0.1:80", "http://127.0.0.1"},
|
||||
{"http://127.0.0.1:0081/x", "http://127.0.0.1:0081/x"},
|
||||
{"http://127.0.0.1:81/x", "http://127.0.0.1:81/x"},
|
||||
{"http://127.0.0.1:8080/x", "http://127.0.0.1:8080/x"},
|
||||
{"http://127.0.0.1:4294967376/x", "http://127.0.0.1:4294967376/x"},
|
||||
{"http://127.0.0.1/x", "http://127.0.0.1/x"},
|
||||
};
|
||||
|
||||
size_t k;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
for (k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
|
||||
char BIGSTK buff[HTS_URLMAXSIZE * 2];
|
||||
|
||||
strcpybuff(buff, cases[k].in);
|
||||
hts_strip_default_port(buff, sizeof(buff));
|
||||
assertf(strcmp(buff, cases[k].out) == 0);
|
||||
}
|
||||
printf("stripport self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
|
||||
no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
|
||||
registering an already-crawled link (dedup/collision paths). */
|
||||
@@ -2111,6 +2146,52 @@ static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Drives unzRepair over a damaged local file header whose CRC field's high
|
||||
16-bit word has bit 15 set. Before the READ_32 fix that shifted an int and
|
||||
overflowed, so UBSan aborts here; after it, repair recovers the one entry. */
|
||||
static int st_zip_repair_shift(httrackp *opt, int argc, char **argv) {
|
||||
static const unsigned char zip[] = {
|
||||
0x50, 0x4b, 0x03, 0x04, /* local file header signature */
|
||||
0x14, 0x00, /* version needed */
|
||||
0x00, 0x00, /* general purpose flag */
|
||||
0x00, 0x00, /* method */
|
||||
0x00, 0x00, /* time */
|
||||
0x00, 0x00, /* date */
|
||||
0x00, 0x00, 0xe8, 0x8a, /* crc: high word 0x8ae8, bit 15 set */
|
||||
0x00, 0x00, 0x00, 0x00, /* compressed size */
|
||||
0x00, 0x00, 0x00, 0x00, /* uncompressed size */
|
||||
0x01, 0x00, /* filename length */
|
||||
0x00, 0x00, /* extra field length */
|
||||
0x61 /* filename "a" */
|
||||
};
|
||||
char in[HTS_URLMAXSIZE], out[HTS_URLMAXSIZE], tmp[HTS_URLMAXSIZE];
|
||||
uLong nrec = 0, bytes = 0;
|
||||
FILE *fp;
|
||||
int err;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "zip-repair-shift: needs a directory\n");
|
||||
return 1;
|
||||
}
|
||||
snprintf(in, sizeof(in), "%s/damaged.zip", argv[0]);
|
||||
snprintf(out, sizeof(out), "%s/repair.zip", argv[0]);
|
||||
snprintf(tmp, sizeof(tmp), "%s/repair.tmp", argv[0]);
|
||||
fp = fopen(in, "wb");
|
||||
if (fp == NULL || fwrite(zip, 1, sizeof(zip), fp) != sizeof(zip)) {
|
||||
if (fp != NULL)
|
||||
fclose(fp);
|
||||
fprintf(stderr, "zip-repair-shift: cannot write %s\n", in);
|
||||
return 1;
|
||||
}
|
||||
fclose(fp);
|
||||
err = unzRepair(in, out, tmp, &nrec, &bytes);
|
||||
printf("zip-repair-shift: %s (recovered %lu entr%s)\n",
|
||||
(err == Z_OK && nrec == 1) ? "OK" : "FAIL", (unsigned long) nrec,
|
||||
nrec == 1 ? "y" : "ies");
|
||||
return (err == Z_OK && nrec == 1) ? 0 : 1;
|
||||
}
|
||||
|
||||
static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
|
||||
int err;
|
||||
|
||||
@@ -3189,6 +3270,8 @@ static const struct selftest_entry {
|
||||
st_socks5},
|
||||
{"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
|
||||
st_identabs},
|
||||
{"stripport", "", "default :80 port strip preserves host (#627)",
|
||||
st_stripport},
|
||||
{"header", "<raw-header-line> ...", "response header-line parsing",
|
||||
st_header},
|
||||
{"headerlong", "[header-name:]",
|
||||
@@ -3216,6 +3299,9 @@ static const struct selftest_entry {
|
||||
st_cache_legacy},
|
||||
{"cache-corrupt", "<dir>", "cache read-side corruption self-test",
|
||||
st_cache_corrupt},
|
||||
{"zip-repair-shift", "<dir>",
|
||||
"cache zip-repair header read must not overflow a signed shift",
|
||||
st_zip_repair_shift},
|
||||
{"dns", "", "DNS resolver/cache self-test", st_dns},
|
||||
{"dnstimeout", "", "a slow DNS resolve is bounded and holds no lock",
|
||||
st_dnstimeout},
|
||||
|
||||
@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htsnet.h"
|
||||
#include "htslib.h"
|
||||
#include "htscharset.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -749,7 +750,14 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
RUN THE SERVER
|
||||
*/
|
||||
if (strcmp((char *) adrcd, "start") == 0) {
|
||||
webhttrack_main((char *) adr + p);
|
||||
/* POST body is in the form's charset, not the
|
||||
UTF-8 argv the engine now assumes (#629). */
|
||||
char *const cmdl = (char *) adr + p;
|
||||
char *cmdlUtf8 = hts_convertStringToUTF8(
|
||||
cmdl, strlen(cmdl), LANGSEL("LANGUAGE_CHARSET"));
|
||||
|
||||
webhttrack_main(cmdlUtf8 != NULL ? cmdlUtf8 : cmdl);
|
||||
freet(cmdlUtf8);
|
||||
} else {
|
||||
commandRunning = 0;
|
||||
commandEnd = 1;
|
||||
|
||||
@@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsserver.h"
|
||||
#include "htsurlport.h"
|
||||
#include "htsweb.h"
|
||||
#include "htscharset.h"
|
||||
|
||||
#if USE_BEGINTHREAD==0
|
||||
#error fatal: no threads support
|
||||
@@ -156,6 +157,7 @@ int main(int argc, char *argv[]) {
|
||||
printf("Initializing the server..\n");
|
||||
|
||||
#ifdef _WIN32
|
||||
hts_argv_utf8(&argc, &argv);
|
||||
{
|
||||
WORD wVersionRequested; // requested version WinSock API
|
||||
WSADATA wsadata; // Windows Sockets API data
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
#define READ_8(adr) ((unsigned char)*(adr))
|
||||
#define READ_16(adr) ( READ_8(adr) | (READ_8(adr+1) << 8) )
|
||||
#define READ_32(adr) ( READ_16(adr) | (READ_16((adr)+2) << 16) )
|
||||
#define READ_32(adr) ((uLong) READ_16(adr) | ((uLong) READ_16((adr) + 2) << 16))
|
||||
|
||||
#define WRITE_8(buff, n) do { \
|
||||
*((unsigned char*)(buff)) = (unsigned char) ((n) & 0xff); \
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"$schema":
|
||||
"https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
|
||||
"$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
|
||||
"name": "libhttrack",
|
||||
"version-string": "3.49",
|
||||
"builtin-baseline": "cd61e1e26a038e82d6550a3ebbe0fbbfe7da78e3",
|
||||
"dependencies": [
|
||||
"brotli",
|
||||
"openssl",
|
||||
"zlib",
|
||||
"zstd"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -9,21 +9,23 @@ set -euo pipefail
|
||||
# drives both refusal paths and the accept path.
|
||||
|
||||
out="$(httrack -O /dev/null -#test=xfread-limit)"
|
||||
# Match with here-strings, not `echo | grep -q`: under pipefail the early grep
|
||||
# exit SIGPIPEs echo and fails the pipeline even when the pattern matched.
|
||||
for case in bylen bygrow; do
|
||||
echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
|
||||
grep -q "${case}: refused=1 adr=null msg=In-memory content too large" <<<"$out" || {
|
||||
echo "FAIL ${case}: $out"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
# Exactly INT32_MAX must be refused too (the reallocs add 1).
|
||||
echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
|
||||
grep -q 'boundary: msg=In-memory content too large' <<<"$out" || {
|
||||
echo "FAIL boundary: $out"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The guard must NOT fire for a legitimate small size.
|
||||
if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
|
||||
if grep -q 'accept: msg=In-memory content too large' <<<"$out"; then
|
||||
echo "FAIL accept (guard fired on a legit size): $out"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
21
tests/01_zlib-repair-shift.test
Normal file
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Keep this POSIX-portable: the harness runs it via $(BASH), which is a plain
|
||||
# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
|
||||
# tool flags despite the #!/bin/bash above.
|
||||
|
||||
# unzRepair header read must not overflow a signed shift (-#test=zip-repair-shift
|
||||
# <dir>). A damaged local file header whose CRC high word has bit 15 set made
|
||||
# READ_32 shift an int past INT_MAX; UBSan aborts before the fix casts to uLong.
|
||||
|
||||
set -eu
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
out=$(httrack -#test=zip-repair-shift "$dir")
|
||||
|
||||
printf '%s\n' "$out" | grep -qx "zip-repair-shift: OK (recovered 1 entry)" || {
|
||||
echo "expected 'zip-repair-shift: OK (recovered 1 entry)', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
11
tests/66_engine-port80-strip.test
Executable file
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Stripping a default :80 from a crawled link used to skip a hardcoded 3 chars
|
||||
# (#627): ":080"/":0080" lost only ":80" and glued the rest onto the host
|
||||
# (127.0.0.1:080/x -> 127.0.0.10/x), and a value wrapping to 80 as a 32-bit int
|
||||
# (#614) was stripped as if it were the default. All assertions live in the
|
||||
# engine self-test.
|
||||
httrack -O /dev/null -#test=stripport | grep -q "stripport self-test OK"
|
||||
72
tests/67_engine-delayed-truncate.test
Executable file
@@ -0,0 +1,72 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# #623: url_savename enforces the 236-char path ceiling by cutting the tail of
|
||||
# the last segment, where the mandatory ".delayed" placeholder marker lives. A
|
||||
# cut marker fails IS_DELAYED_EXT, so back_delayed_rename never renames the file
|
||||
# to its final name and the download is lost. The marker must survive the cut.
|
||||
|
||||
# statuscode=302 status=-1 = a redirect answer still downloading: no type is
|
||||
# resolved, so the name gets a ".<id>.delayed" placeholder (see 01_engine-savename).
|
||||
CEIL=236
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
cd "$scratch"
|
||||
|
||||
run() {
|
||||
"$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
|
||||
}
|
||||
|
||||
# A deep path ending in a long segment that overruns the ceiling.
|
||||
deep="/d1$(printf 'a%.0s' {1..40})/d2$(printf 'b%.0s' {1..40})"
|
||||
deep="$deep/d3$(printf 'c%.0s' {1..40})/d4$(printf 'd%.0s' {1..40})"
|
||||
long="$deep/$(printf 'z%.0s' {1..90})"
|
||||
|
||||
out="$(run "$long" text/html statuscode=302 status=-1)"
|
||||
case "$out" in
|
||||
*.delayed) ;;
|
||||
*)
|
||||
echo "FAIL: delayed marker cut by truncation: '$out'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
test "${#out}" -le "$CEIL" || {
|
||||
echo "FAIL: truncated name ${#out} > $CEIL ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# #133-style hashed name: an all-hex last segment must keep the marker too (the
|
||||
# ".<id>." tag is not mistaken for part of the hash).
|
||||
hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8})
|
||||
hexpath="/d1$(printf 'x%.0s' {1..30})/d2$(printf 'y%.0s' {1..30})/$hexseg"
|
||||
out="$(run "$hexpath" text/html statuscode=302 status=-1)"
|
||||
case "$out" in
|
||||
*.delayed) ;;
|
||||
*)
|
||||
echo "FAIL: hashed-name marker cut by truncation: '$out'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
test "${#out}" -le "$CEIL" || {
|
||||
echo "FAIL: hashed name ${#out} > $CEIL ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# A non-delayed name of the same shape still truncates, with no marker to keep.
|
||||
out="$(run "$long.html" text/html)"
|
||||
test "${#out}" -le "$CEIL" || {
|
||||
echo "FAIL: non-delayed name ${#out} > $CEIL ceiling: '$out'"
|
||||
exit 1
|
||||
}
|
||||
case "$out" in
|
||||
*.delayed)
|
||||
echo "FAIL: non-delayed name grew a .delayed marker: '$out'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "delayed-truncate OK"
|
||||
90
tests/68_webhttrack-outdir-charset.test
Executable file
@@ -0,0 +1,90 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# webhttrack's POST body arrives in the form's declared charset, not UTF-8, yet
|
||||
# the engine now assumes UTF-8 argv. A non-ASCII -O output dir submitted through
|
||||
# the web UI must land under the UTF-8 directory, not an ISO-8859-1 twin (#629).
|
||||
# Drives the real htsserver over HTTP; the default (English) form is served
|
||||
# ISO-8859-1, so 'café' travels as the single byte 0xE9. The crawl target is a
|
||||
# dead port: only the -O directory name is under test, and htsserver creates it
|
||||
# from the decoded path before any fetch.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
|
||||
distdir=$(cd "${distdir}" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v htsserver >/dev/null || fail "no htsserver in PATH"
|
||||
python=$(find_python) || {
|
||||
echo "python3 not found; skipping" >&2
|
||||
exit 77
|
||||
}
|
||||
|
||||
work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_charset.XXXXXX") || fail "no tmpdir"
|
||||
srvlog=$(mktemp)
|
||||
srv=
|
||||
cleanup() {
|
||||
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
|
||||
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
|
||||
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
|
||||
rm -rf "${work}" "${srvlog}"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
|
||||
# ~/.httrack.ini out of it.
|
||||
sport=$("${python}" -c 'import socket
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
s.close()')
|
||||
(
|
||||
trap '' TERM TTOU
|
||||
export HOME="${work}"
|
||||
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
|
||||
) &
|
||||
srv=$!
|
||||
for _ in $(seq 1 40); do
|
||||
url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
|
||||
kill -0 "${srv}" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
|
||||
|
||||
# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset.
|
||||
"${python}" - "${url}" "${work}" <<'PY'
|
||||
import sys, urllib.parse, urllib.request
|
||||
|
||||
url, work = sys.argv[1], sys.argv[2]
|
||||
outdir = work + "/caf\xe9" # 'café' as the single byte the browser would send
|
||||
# Port 1 refuses at once: only the decoded -O dir matters, not the fetch.
|
||||
cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir
|
||||
fields = [("path", work), ("projname", "proj"), ("command_do", "start"),
|
||||
("winprofile", "x"), ("command", cmd)]
|
||||
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1"))
|
||||
for k, v in fields)
|
||||
req = urllib.request.Request(url + "step4.html", data=body.encode("latin-1"),
|
||||
method="POST")
|
||||
urllib.request.urlopen(req, timeout=20).read()
|
||||
PY
|
||||
|
||||
# The crawl runs inside htsserver; wait for the -O dir, then check it chose the
|
||||
# UTF-8 name and not the ISO-8859-1 twin.
|
||||
utf8dir="${work}/café"
|
||||
latin1dir="${work}/caf"$'\xe9'
|
||||
for _ in $(seq 1 80); do
|
||||
test -e "${utf8dir}" && break
|
||||
kill -0 "${srv}" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
test -e "${utf8dir}" || fail "-O dir never landed as UTF-8 café/: $(find "${work}" -maxdepth 2 2>/dev/null)"
|
||||
test ! -e "${latin1dir}" || fail "-O dir created as the ISO-8859-1 twin instead"
|
||||
|
||||
echo "PASS"
|
||||
15
tests/69_local-intl-logdir.test
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The
|
||||
# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin:
|
||||
# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage.
|
||||
# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so
|
||||
# a log written to the twin fails them. POSIX has no twin, so this bites on the
|
||||
# Windows CI leg (like test 64). The cache twin is a separate, larger fix.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \
|
||||
--found 'simple/basic.html' \
|
||||
--log-found 'mirror complete in' \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
134
tests/71_local-crange-repaircache.test
Executable file
@@ -0,0 +1,134 @@
|
||||
#!/bin/bash
|
||||
# #581: an interrupted mirror can leave a truncated new.zip that pass 2 must
|
||||
# repair before resuming. The repaired-cache resume then hits the same hostile
|
||||
# 206 as test 48, so restart-whole must still drop the partial and refetch the
|
||||
# whole file. Pass 1 leaves a partial + temp-ref; we truncate new.zip past its
|
||||
# last local entry (dropping the central directory) so unzOpen fails and the
|
||||
# repair path runs; pass 2 resumes, rejects the range, and refetches whole.
|
||||
set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
root=$(nativepath "${testdir}/server-root")
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
# On Windows the pass-1 interrupt is a hard kill (MSYS can't signal a native
|
||||
# exe) and the restart-whole path fails on the repaired cache (#581) -- the very
|
||||
# bug this exercises; skip until the engine fix lands.
|
||||
if is_windows; then
|
||||
echo "Windows: restart-whole fails on a repaired cache (#581), skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangerep.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
|
||||
refdir="${out}/hts-cache/ref"
|
||||
newzip="${out}/hts-cache/new.zip"
|
||||
|
||||
# --- pass 1: crawl, interrupt once the blob download is underway -------------
|
||||
printf '[pass 1: interrupt mid-download] ..\t'
|
||||
httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
|
||||
crawlpid=$!
|
||||
for _ in $(seq 1 300); do
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
|
||||
kill -0 "$crawlpid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
sleep 0.3
|
||||
kill -TERM "$crawlpid" 2>/dev/null
|
||||
wait "$crawlpid" 2>/dev/null
|
||||
crawlpid=
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
|
||||
echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (temp-ref present)"
|
||||
|
||||
# --- damage the cache: drop the central directory a hard kill never wrote ----
|
||||
printf '[damage new.zip -> forces repair] ..\t'
|
||||
"$python" - "$newzip" <<'PY' || exit 1
|
||||
import os, sys
|
||||
path = sys.argv[1]
|
||||
data = open(path, "rb").read()
|
||||
cut = data.find(b"PK\x01\x02") # first central-directory header
|
||||
if cut <= 0:
|
||||
sys.exit("no central directory to drop in %s" % path)
|
||||
os.truncate(path, cut)
|
||||
PY
|
||||
echo "OK"
|
||||
|
||||
# --- pass 2: --continue -> repair -> resume -> hostile 206 -> refetch whole ---
|
||||
printf '[pass 2: repair, reject range, refetch] ..\t'
|
||||
rc=0
|
||||
httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1 || rc=$?
|
||||
test "$rc" -eq 0 || {
|
||||
echo "FAIL: httrack exited $rc"
|
||||
cat "${tmpdir}/log2" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK (terminated)"
|
||||
|
||||
# The repair path must actually have run, else this is just a copy of test 48.
|
||||
printf '[cache repair fired] ..\t'
|
||||
grep -q 'damaged cache' "${out}/hts-log.txt" 2>/dev/null || {
|
||||
echo "FAIL: repair path did not run; damage was ineffective"
|
||||
exit 1
|
||||
}
|
||||
echo "OK"
|
||||
|
||||
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
|
||||
full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
|
||||
|
||||
printf '[file recovered whole] ..\t'
|
||||
test -s "$blob" || {
|
||||
echo "FAIL: blob.bin missing after the repaired-cache resume"
|
||||
exit 1
|
||||
}
|
||||
got=$(wc -c <"$blob")
|
||||
test "$got" -eq "$full" || {
|
||||
echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (${got} bytes)"
|
||||
@@ -84,6 +84,7 @@ TESTS = \
|
||||
01_zlib-cache-legacy.test \
|
||||
01_zlib-cache-golden.test \
|
||||
01_zlib-cache-writefail.test \
|
||||
01_zlib-repair-shift.test \
|
||||
01_zlib-savename-cached.test \
|
||||
02_manpage-regen.test \
|
||||
02_update-cache.test \
|
||||
@@ -146,6 +147,11 @@ TESTS = \
|
||||
62_lang-integrity.test \
|
||||
63_webhttrack-home.test \
|
||||
64_local-intl-outdir.test \
|
||||
65_port-siblings.test
|
||||
65_port-siblings.test \
|
||||
66_engine-port80-strip.test \
|
||||
67_engine-delayed-truncate.test \
|
||||
68_webhttrack-outdir-charset.test \
|
||||
69_local-intl-logdir.test \
|
||||
71_local-crange-repaircache.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -48,6 +48,7 @@ key="${testdir}/server.key"
|
||||
tls=
|
||||
verbose=
|
||||
html_subdir=
|
||||
outdir_intl=
|
||||
rerun=
|
||||
rerun_args=
|
||||
rerun_dead=
|
||||
@@ -140,6 +141,13 @@ while test "$pos" -lt "$nargs"; do
|
||||
pos=$((pos + 1))
|
||||
html_subdir="${args[$pos]}"
|
||||
;;
|
||||
--outdir-intl)
|
||||
# Single non-ASCII -O "$out/NAME": path_html AND path_log are NAME, so
|
||||
# the logs (and the harness reads of them) go through the non-ASCII path
|
||||
# (#630). Distinct from --html-subdir, which keeps path_log ASCII.
|
||||
pos=$((pos + 1))
|
||||
outdir_intl="${args[$pos]}"
|
||||
;;
|
||||
--errors | --errors-content | --files)
|
||||
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
|
||||
pos=$((pos + 1))
|
||||
@@ -216,13 +224,18 @@ test -n "$ver" || die "could not run httrack"
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out" || die "could not create $out"
|
||||
# path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
|
||||
# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME"
|
||||
# (the mirror root the audits inspect) while path_log stays "$out".
|
||||
# Default: both are "$out". --html-subdir moves path_html to "$out/NAME" while
|
||||
# path_log (logroot) stays "$out"; --outdir-intl moves both to "$out/NAME".
|
||||
mirrorroot="$out"
|
||||
logroot="$out"
|
||||
odir="$out"
|
||||
if test -n "$html_subdir"; then
|
||||
mirrorroot="${out}/${html_subdir}"
|
||||
odir="${mirrorroot},${out}"
|
||||
elif test -n "$outdir_intl"; then
|
||||
mirrorroot="${out}/${outdir_intl}"
|
||||
logroot="$mirrorroot"
|
||||
odir="$mirrorroot"
|
||||
fi
|
||||
# Localhost is fast; disable the rate/bandwidth safety limits but keep a
|
||||
# max-time backstop so a hang cannot wedge the suite.
|
||||
@@ -241,7 +254,7 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
|
||||
exit 1
|
||||
}
|
||||
result "OK"
|
||||
grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
|
||||
grep -iE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt" >&2
|
||||
|
||||
# --- optional second pass: re-mirror into the same dir (cache/update path) ----
|
||||
if test -n "$rerun"; then
|
||||
@@ -260,7 +273,7 @@ if test -n "$rerun"; then
|
||||
# The update summary reports "files updated"; a fresh crawl never does. Assert
|
||||
# it so a regression that bypasses the cache (re-crawls fresh) can't pass.
|
||||
info "checking update used the cache"
|
||||
if grep -aqE "mirror complete in .*files updated" "${out}/hts-log.txt"; then
|
||||
if grep -aqE "mirror complete in .*files updated" "${logroot}/hts-log.txt"; then
|
||||
result "OK"
|
||||
else
|
||||
result "update pass did not report cache activity"
|
||||
@@ -292,7 +305,7 @@ if test -n "$rerun_dead"; then
|
||||
zip="${out}/hts-cache/new.zip"
|
||||
test -s "$zip" || die "no cache was written by the first pass"
|
||||
cp "$zip" "${tmpdir}/cache-before.zip"
|
||||
cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
|
||||
cp "${logroot}/hts-log.txt" "${tmpdir}/log-before.txt"
|
||||
stop_server "$serverpid"
|
||||
serverpid=
|
||||
info "re-running httrack against the stopped server"
|
||||
@@ -305,7 +318,7 @@ if test -n "$rerun_dead"; then
|
||||
# The dead pass must have gone through the no-data rollback, not bailed out
|
||||
# before the mirror loop (which would leave the cache trivially untouched).
|
||||
info "checking the dead pass hit the rollback"
|
||||
if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
|
||||
if grep -aq "No data seems to have been transferred" "${logroot}/hts-log.txt"; then
|
||||
result "OK"
|
||||
else
|
||||
result "rollback notice not found in hts-log.txt"
|
||||
@@ -320,7 +333,7 @@ if test -n "$rerun_dead"; then
|
||||
exit 1
|
||||
fi
|
||||
# Audits below describe the healthy crawl, not the dead pass.
|
||||
cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
|
||||
cp "${tmpdir}/log-before.txt" "${logroot}/hts-log.txt"
|
||||
fi
|
||||
|
||||
# --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
|
||||
@@ -350,19 +363,19 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
--errors)
|
||||
i=$((i + 1))
|
||||
assert_equals "checking errors" "${audit[$i]}" \
|
||||
"$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
|
||||
"$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")"
|
||||
;;
|
||||
--errors-content)
|
||||
i=$((i + 1))
|
||||
total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
|
||||
total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")
|
||||
# transient network failures (statuscode -2..-6) flake on busy loopback;
|
||||
# the code parens are followed by " at link" or " after N retries at link"
|
||||
transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
|
||||
transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${logroot}/hts-log.txt" || true)
|
||||
assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
|
||||
;;
|
||||
--files)
|
||||
i=$((i + 1))
|
||||
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
|
||||
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${logroot}/hts-log.txt" |
|
||||
sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
|
||||
assert_equals "checking files" "${audit[$i]}" "$nFiles"
|
||||
;;
|
||||
@@ -393,7 +406,7 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
--log-found)
|
||||
i=$((i + 1))
|
||||
info "checking log matches ${audit[$i]}"
|
||||
if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then result "OK"; else
|
||||
if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then result "OK"; else
|
||||
result "not in log"
|
||||
exit 1
|
||||
fi
|
||||
@@ -401,7 +414,7 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
--log-not-found)
|
||||
i=$((i + 1))
|
||||
info "checking log lacks ${audit[$i]}"
|
||||
if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then
|
||||
if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then
|
||||
result "present in log"
|
||||
exit 1
|
||||
else result "OK"; fi
|
||||
|
||||