mirror of
https://github.com/xroche/httrack.git
synced 2026-07-22 16:51:25 +03:00
Compare commits
9 Commits
fix-help-N
...
retire-cmd
| Author | SHA1 | Date | |
|---|---|---|---|
|
4de1b170b6 | ||
|
|
d2e94b1c99 | ||
|
|
91071cb003 | ||
|
|
2b5e740b55 | ||
|
|
1d62527d2d | ||
|
|
0d25999069 | ||
|
|
8904656164 | ||
|
|
86b46947fb | ||
|
|
71c1764525 |
28
.github/workflows/ci.yml
vendored
28
.github/workflows/ci.yml
vendored
@@ -618,3 +618,31 @@ jobs:
|
||||
echo "Fix locally with: git clang-format --binary clang-format-19 $base"
|
||||
exit 1 ;;
|
||||
esac
|
||||
|
||||
man-page-sync:
|
||||
name: man page / html in sync
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# html/httrack.man.html is groff-rendered from man/httrack.1 and committed.
|
||||
# Rendering needs the full groff html device, so CI can't regenerate it;
|
||||
# instead require the two to move together: a PR that touches httrack.1
|
||||
# must also touch the html, catching the "regenerated roff, forgot html".
|
||||
- name: httrack.1 changes must include html/httrack.man.html
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
|
||||
base="origin/${{ github.base_ref }}"
|
||||
changed="$(git diff --name-only "$base"...HEAD)"
|
||||
has() { printf '%s\n' "$changed" | grep -qx "$1"; }
|
||||
if has man/httrack.1 && ! has html/httrack.man.html; then
|
||||
echo "::error::man/httrack.1 changed but html/httrack.man.html did not."
|
||||
echo "Regenerate it with: make -C man regen-man-html (needs the full groff package)."
|
||||
exit 1
|
||||
fi
|
||||
echo "man/html sync OK."
|
||||
|
||||
3
.github/workflows/codeql.yml
vendored
3
.github/workflows/codeql.yml
vendored
@@ -55,6 +55,9 @@ jobs:
|
||||
query-filters:
|
||||
- exclude:
|
||||
id: cpp/world-writable-file-creation
|
||||
# Models auth-bypass-by-spoofing; httrack has no auth surface, its +/- crawl filter is a mirror boundary, not a security one.
|
||||
- exclude:
|
||||
id: cpp/user-controlled-bypass
|
||||
|
||||
# Manual build: CodeQL traces the compiler, so build exactly what ships.
|
||||
- name: Build
|
||||
|
||||
157
html/cmddoc.html
157
html/cmddoc.html
@@ -1,157 +0,0 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>Command-Line Documentation</em></h2>
|
||||
|
||||
<br>
|
||||
|
||||
The command-line version
|
||||
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
<br>A task-oriented guide: how to do the common things, and the defaults that surprise newcomers<br><br>
|
||||
<li><a href="options.html">Command line Options</a></li>
|
||||
<br>List of all powerful command line options<br><br>
|
||||
<li>How to use httrack command-line version:</li>
|
||||
<ul>
|
||||
<li>Open a shell window</li>
|
||||
<br>
|
||||
<br>
|
||||
<li>Type in <tt>httrack</tt> (or the complete path to the httrack executable)</li>
|
||||
<br><small><tt>httrack</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Add the URLs, separated by a blank space</li>
|
||||
<br><small><tt>httrack www.example.com/foo/</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>If you need, add some options (see the <a href="options.html">option list</a>)</li>
|
||||
<br><small><tt>httrack www.example.com/foo/ -O "/webs" -N4 -P proxy.myhost.com:3128</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Launch the command line, and wait until the mirror is finishing</li>
|
||||
<br><small>You can (especially on the Unix release) press ^C to stop the mirror or put httrack in background</small>
|
||||
<br>
|
||||
<br>
|
||||
</ul>
|
||||
</ul>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
@@ -219,6 +219,23 @@ that a <tt>403 Forbidden</tt> is a server refusal, not a robots rule: robots
|
||||
options will not help there. That is an
|
||||
<a href="#identity">identity</a> problem.</p>
|
||||
|
||||
<h4>Filter wildcards</h4>
|
||||
<p>Inside a filter pattern, <tt>*</tt> matches any run of characters; a few
|
||||
bracket forms match narrower sets. The full table, with size and mime rules, is on
|
||||
<a href="filters.html">the filters page</a>.</p>
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Wildcard</b></td><td><b>Matches</b></td><td><b>Example</b></td></tr>
|
||||
<tr><td><tt>*</tt></td><td>any run of characters</td><td><tt>+*.pdf</tt> — any URL ending <tt>.pdf</tt></td></tr>
|
||||
<tr><td><tt>*[file]</tt>, <tt>*[name]</tt></td><td>one path segment (any char but <tt>/</tt> and <tt>?</tt>)</td><td><tt>example.com/*[file]/</tt> — a directory-index page</td></tr>
|
||||
<tr><td><tt>*[path]</tt></td><td>a path, slashes allowed (any char but <tt>?</tt>)</td><td><tt>example.com/*[path].zip</tt></td></tr>
|
||||
<tr><td><tt>*[param]</tt></td><td>an optional query string</td><td><tt>page.html*[param]</tt> matches with or without <tt>?...</tt></td></tr>
|
||||
<tr><td><tt>*[a,b,c]</tt></td><td>any one character in the set</td><td><tt>*[a,b,c].txt</tt></td></tr>
|
||||
<tr><td><tt>*[a-z]</tt></td><td>any one character in the range</td><td><tt>img*[0-9].gif</tt></td></tr>
|
||||
<tr><td><tt>*[\x]</tt></td><td>the literal character x (escapes <tt>* [ ] \</tt>)</td><td><tt>*[\*]</tt> matches a real <tt>*</tt></td></tr>
|
||||
<tr><td><tt>*[<NN]</tt>, <tt>*[>NN]</tt></td><td>file size in KB below / above NN</td><td><tt>-*.gif*[<5]</tt> skips GIFs under 5 KB</td></tr>
|
||||
<tr><td><tt>*[]</tt></td><td>end anchor: nothing may follow</td><td><tt>*.html*[]</tt> rejects <tt>i.html?p=1</tt></td></tr>
|
||||
</table>
|
||||
|
||||
<h3 id="limits">4. Limits and politeness</h3>
|
||||
|
||||
<p>HTTrack ships cautious on purpose: it is easy to hammer a small site by
|
||||
@@ -234,7 +251,7 @@ below let you go faster when you own the target, and slower when you do not.</p>
|
||||
<tr><td><tt>--max-time (-E)</tt></td><td>Stop after N seconds of wall-clock time.</td></tr>
|
||||
<tr><td><tt>--max-files (-m)</tt></td><td>Per-file size caps.</td></tr>
|
||||
<tr><td><tt>--timeout (-T), --retries (-R), --min-rate (-J), --host-control (-H)</tt></td><td>Idle timeout, retry count, minimum acceptable rate, and host-ban behavior for slow or dead hosts.</td></tr>
|
||||
<tr><td><tt>--max-pause (-G), -%G</tt></td><td>Pause the mirror at N bytes, or pause between files, to spread the load.</td></tr>
|
||||
<tr><td><tt>--max-pause (-G), --pause (-%G)</tt></td><td>Pause the mirror at N bytes, or pause between files, to spread the load.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>The security clamps.</b> To keep an accidental typo from turning into a flood,
|
||||
@@ -257,8 +274,8 @@ really HTML.</p>
|
||||
<tr><td><tt>--structure (-N)</tt></td><td>The local path and name layout. Presets are numeric, and you can also give a template such as <tt>--structure "%h%p/%n%q.%t"</tt>.</td></tr>
|
||||
<tr><td><tt>--long-names (-L)</tt></td><td>Long names, 8.3 names, or ISO9660 for CD masters.</td></tr>
|
||||
<tr><td><tt>--assume (-%A)</tt></td><td>Assume a MIME type for an extension, for example <tt>--assume php=text/html</tt>. This also skips the extra HEAD probe HTTrack would otherwise send to learn the type.</td></tr>
|
||||
<tr><td><tt>-%N, --cached-delayed-type-check (-%D), --check-type (-u), -%t</tt></td><td>When and how the content type is checked, and whether the original extension is kept.</td></tr>
|
||||
<tr><td><tt>--include-query-string (-%q), -%g</tt></td><td>Whether the query string appears in the local filename, and whether query keys are stripped when deciding if two URLs are the same file.</td></tr>
|
||||
<tr><td><tt>--delayed-type-check (-%N), --cached-delayed-type-check (-%D), --check-type (-u), -%t</tt></td><td>When and how the content type is checked, and whether the original extension is kept.</td></tr>
|
||||
<tr><td><tt>--include-query-string (-%q), --strip-query (-%g)</tt></td><td>Whether the query string appears in the local filename, and whether query keys are stripped when deciding if two URLs are the same file.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>The <tt>-N</tt> presets are built from modular arithmetic on the name fields, so
|
||||
@@ -329,7 +346,7 @@ across links at the same time.</p>
|
||||
<tr><td><tt>--proxy (-P)</tt></td><td>Route through a proxy. HTTP, SOCKS5 and CONNECT are supported: <tt>-P host:8080</tt>, <tt>-P socks5://host:1080</tt>, <tt>-P connect://host:443</tt>, with optional <tt>user:pass@</tt>.</td></tr>
|
||||
<tr><td><tt>--httpproxy-ftp (-%f)</tt></td><td>Send FTP requests through the HTTP proxy.</td></tr>
|
||||
<tr><td><tt>--protocol (-@i)</tt></td><td>Prefer IPv4 or IPv6.</td></tr>
|
||||
<tr><td><tt>--http-10 (-%h), --keep-alive (-%k), -%z</tt></td><td>Force HTTP/1.0 (drops keep-alive and compression, useful for fragile CGI), toggle keep-alive, and toggle compression.</td></tr>
|
||||
<tr><td><tt>--http-10 (-%h), --keep-alive (-%k), --disable-compression (-%z)</tt></td><td>Force HTTP/1.0 (drops keep-alive and compression, useful for fragile CGI), toggle keep-alive, and toggle compression.</td></tr>
|
||||
<tr><td><tt>--bind (-%b), --tolerant (-%B)</tt></td><td>Bind to a local address, and accept technically-bogus responses some servers send.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -353,7 +370,7 @@ away and you lose the ability to continue or update the mirror.</p>
|
||||
<tr><td><tt>--update</tt></td><td>Re-run the mirror, revalidating each page with the server (If-Modified-Since / If-None-Match) and downloading only what changed.</td></tr>
|
||||
<tr><td><tt>--purge-old=0 (-X0)</tt></td><td>Do not purge. By default an update deletes local files that are no longer part of the mirror; <tt>--purge-old=0</tt> keeps them.</td></tr>
|
||||
<tr><td><tt>--cache (-C)</tt></td><td>Cache mode. The default already does the right thing and switches to update-checking when it detects an existing mirror.</td></tr>
|
||||
<tr><td><tt>--urlhack (-%u), -%j, -%o, -%y, --do-not-recatch (-%n), --updatehack (-%s), --store-all-in-cache (-k)</tt></td><td>URL-deduplication behavior and cache storage details.</td></tr>
|
||||
<tr><td><tt>--urlhack (-%u), --keep-www-prefix (-%j), --keep-double-slashes (-%o), --keep-query-order (-%y), --do-not-recatch (-%n), --updatehack (-%s), --store-all-in-cache (-k)</tt></td><td>URL-deduplication behavior and cache storage details.</td></tr>
|
||||
<tr><td><tt>--debug-cache (-#C), --repair-cache (-#R), --clean</tt></td><td>Inspect the cache, repair its ZIP, and erase cache plus logs.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -404,14 +421,16 @@ host and stops the crawl; start from the final URL, or add
|
||||
rule wins.</small></p>
|
||||
|
||||
<h4>Download the PDFs on a site</h4>
|
||||
<p><tt>httrack https://example.com/ --path mydir</tt><br>
|
||||
<small>There is no PDF-only crawl. HTTrack discovers PDF links by parsing the
|
||||
site's HTML pages, so a <tt>"-*" "+example.com/*.pdf"</tt> filter blocks the very
|
||||
pages that carry the links and grabs only the PDFs linked from the front page. Let
|
||||
it crawl the site: the HTML pages come along as the scaffolding, and every reachable
|
||||
PDF is saved with them. If some PDFs live on another host (a CDN or a docs
|
||||
subdomain), allow that host too, for example
|
||||
<tt>"+docs.example.com/*.pdf"</tt>.</small></p>
|
||||
<p><tt>httrack https://example.com/ "-*" "+https://example.com/*.html" "+https://example.com/*[path]/" "+https://example.com/*.pdf" --path mydir</tt><br>
|
||||
<small>HTTrack finds PDFs by parsing the site's HTML, so a plain
|
||||
<tt>"-*" "+example.com/*.pdf"</tt> is wrong: it prunes the pages that carry the
|
||||
links and keeps only PDFs reachable from the front page. Instead admit the HTML as
|
||||
scaffolding (<tt>*.html</tt> and <tt>*[path]/</tt> for directory-index pages at any
|
||||
depth, e.g. <tt>docs/</tt> or <tt>a/b/deep/</tt>; <tt>*[file]/</tt> would stop at one
|
||||
level), keep the PDFs, and let <tt>-*</tt> drop everything else (images,
|
||||
archives, off-site assets). PDFs on another host (a CDN or docs subdomain) are not
|
||||
included by default; allow that host too, e.g. <tt>"+docs.example.com/*.pdf"</tt>,
|
||||
or widen to <tt>"+*.pdf"</tt> for PDFs anywhere.</small></p>
|
||||
|
||||
<h4>Keep page requisites, including off-host images</h4>
|
||||
<p><tt>httrack https://example.com/blog/ --near --path mydir</tt><br>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -109,12 +109,12 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li>How to Use</li>
|
||||
<ul>
|
||||
<li><a href="shelldoc.html">WinHTTrack/WebHTTrack (GUI version for Windows or Linux/Unix)</a></li>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix relese (WebHTTrack)<br>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix release (WebHTTrack)<br>
|
||||
<br>
|
||||
<li><a href="android.html">HTTrack on Android</a></li>
|
||||
<br>Step-by-step guide for the Android app<br>
|
||||
<br>
|
||||
<li><a href="cmddoc.html">Command-line version</a></li>
|
||||
<li><a href="httrack.man.html">Command-line version</a></li>
|
||||
<br>How to run HTTrack from the command line, and where to find every option<br>
|
||||
<br>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
@@ -123,7 +123,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li><a href="fcguide.html">HTTrack Users Guide By Fred Cohen</a></li>
|
||||
<br>A tutorial that describes all command-line options, for Linux and Windows users<br>
|
||||
<br>
|
||||
<li><a href="dev.html">Developper/Programming</a></li>
|
||||
<li><a href="dev.html">Developer/Programming</a></li>
|
||||
<br>How to use HTTrack in batch files, and how to use the library<br>
|
||||
</ul>
|
||||
<br>
|
||||
|
||||
@@ -101,6 +101,10 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<h2 align="center"><em>Options</em></h2>
|
||||
|
||||
<p><strong>Note:</strong> this is a legacy option list and may be out of date.
|
||||
For the authoritative, always-current reference see <a href="httrack.man.html">the httrack manual page</a>,
|
||||
and for a task-oriented walkthrough see the <a href="cmdguide.html">Command-line Guide</a>.</p>
|
||||
|
||||
<ul>
|
||||
<li>Filters: <a href="filters.html">how to use them</a></li>
|
||||
<br><small>Here you can find informations on filters: how to accept all gif files in a mirror, for example</small>
|
||||
|
||||
@@ -16,6 +16,11 @@ regen-man: makeman.sh $(top_builddir)/src/httrack$(EXEEXT)
|
||||
|
||||
# Render html/httrack.man.html from httrack.1. Needs the groff html device
|
||||
# (Debian: full "groff" package, not "groff-base"). Run by hand: make -C man regen-man-html
|
||||
# Strip groff's version-stamp and creation-date comments so the committed file
|
||||
# doesn't churn across groff versions or rebuilds; the ci man-page-sync guard
|
||||
# only requires this file to change whenever httrack.1 does.
|
||||
regen-man-html: httrack.1
|
||||
groff -t -man -Thtml $(srcdir)/httrack.1 > $(top_srcdir)/html/httrack.man.html
|
||||
groff -t -man -Thtml $(srcdir)/httrack.1 \
|
||||
| sed -e 's/groff version [0-9][0-9.]*/groff/' -e '/^<!-- CreationDate:/d' \
|
||||
> $(top_srcdir)/html/httrack.man.html
|
||||
.PHONY: regen-man-html
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "21 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "22 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -37,6 +37,7 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%L, \-\-list\fR ]
|
||||
[ \fB\-%S, \-\-urllist\fR ]
|
||||
[ \fB\-NN, \-\-structure[=N]\fR ]
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
@@ -189,7 +190,7 @@ structure type (0 *original structure, 1+: see below) (\-\-structure[=N])
|
||||
.br
|
||||
or user defined structure (\-N "%h%p/%n%q.%t")
|
||||
.IP \-%N
|
||||
delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use)
|
||||
delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use) (\-\-delayed\-type\-check)
|
||||
.IP \-%D
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
|
||||
@@ -1617,14 +1617,15 @@ int back_add_if_not_exists(struct_back * sback, httrackp * opt,
|
||||
back_clean(opt, cache, sback); /* first cleanup the backlog to ensure that we have some entry left */
|
||||
if (!back_exist(sback, opt, adr, fil, save)) {
|
||||
return back_add(sback, opt, cache, adr, fil, save, referer_adr, referer_fil,
|
||||
test);
|
||||
test, HTS_FALSE);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, const char *referer_adr, const char *referer_fil,
|
||||
int test) {
|
||||
int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
|
||||
const char *adr, const char *fil, const char *save,
|
||||
const char *referer_adr, const char *referer_fil, int test,
|
||||
hts_boolean refetch_whole) {
|
||||
lien_back *const back = sback->lnk;
|
||||
const int back_max = sback->count;
|
||||
int p = 0;
|
||||
@@ -1701,6 +1702,12 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
else if (strcmp(back[p].url_sav, BACK_ADD_TEST2) == 0) // test en GET
|
||||
back[p].head_request = 2; // test en get
|
||||
|
||||
/* Forced whole refetch (#581): drop the stale temp-ref and skip the resume
|
||||
branches below, so a surviving partial can't Range-loop. */
|
||||
if (refetch_whole) {
|
||||
url_savename_refname_remove(opt, adr, fil);
|
||||
}
|
||||
|
||||
/* Stop requested - abort backing */
|
||||
/* For update mode: second check after cache lookup not to lose all previous cache data ! */
|
||||
if (opt->state.stop && !opt->is_update) {
|
||||
@@ -1956,8 +1963,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
}
|
||||
}
|
||||
/* Not in cache ; maybe in temporary cache ? Warning: non-movable
|
||||
"url_sav" */
|
||||
else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
|
||||
"url_sav" (skipped on a forced whole refetch, #581) */
|
||||
else if (!refetch_whole &&
|
||||
back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
|
||||
const LLint file_size = fsize_utf8(itemback->url_sav);
|
||||
|
||||
/* Found file on disk */
|
||||
@@ -1991,8 +1999,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
freet(itemback); /* delete item */
|
||||
itemback = NULL;
|
||||
}
|
||||
/* Not in cache or temporary cache ; found on disk ? (hack) */
|
||||
else if (fexist_utf8(save)) {
|
||||
/* Not in cache or temporary cache ; found on disk ? (hack)
|
||||
(skipped on a forced whole refetch, #581) */
|
||||
else if (!refetch_whole && fexist_utf8(save)) {
|
||||
const LLint sz = fsize_utf8(save);
|
||||
|
||||
// Bon, là il est possible que le fichier ait été partiellement transféré
|
||||
@@ -3847,6 +3856,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
back[i].r.refetch_wholefile =
|
||||
HTS_TRUE; // retry whole, no Range (#581)
|
||||
strcpybuff(back[i].r.msg,
|
||||
"Bogus 304 on resume, restarting");
|
||||
back[i].status = STATUS_READY;
|
||||
@@ -4118,6 +4129,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
// the resume was rejected: the retry must GET the whole
|
||||
// file, never re-Range a surviving partial/ref (#581)
|
||||
back[i].r.refetch_wholefile = HTS_TRUE;
|
||||
if (strnotempty(back[i].r.msg))
|
||||
strcpybuff(back[i].r.msg,
|
||||
"Error attempting to solve status 206 (partial file)");
|
||||
|
||||
@@ -83,9 +83,12 @@ HTS_INLINE int back_exist(struct_back * sback, httrackp * opt, const char *adr,
|
||||
const char *fil, const char *sav);
|
||||
int back_nsoc(const struct_back * sback);
|
||||
int back_nsoc_overall(const struct_back * sback);
|
||||
int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, const char *referer_adr, const char *referer_fil,
|
||||
int test);
|
||||
/* refetch_whole: force a whole-file GET, ignoring any partial/temp-ref resume
|
||||
(set when a prior 206 was rejected as unusable, #581). */
|
||||
int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
|
||||
const char *adr, const char *fil, const char *save,
|
||||
const char *referer_adr, const char *referer_fil, int test,
|
||||
hts_boolean refetch_whole);
|
||||
int back_add_if_not_exists(struct_back * sback, httrackp * opt,
|
||||
cache_back * cache, const char *adr, const char *fil, const char *save,
|
||||
const char *referer_adr, const char *referer_fil, int test);
|
||||
|
||||
220
src/htscache.c
220
src/htscache.c
@@ -856,7 +856,41 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
// si save==null alors test unqiquement
|
||||
static int hts_rename(httrackp * opt, const char *a, const char *b) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: rename %s -> %s (%p %p)", a, b, a, b);
|
||||
return rename(a, b);
|
||||
return RENAME(a, b);
|
||||
}
|
||||
|
||||
/* Open the cache ZIP via hts_fopen_utf8 so a non-ASCII path_log isn't mangled
|
||||
to ANSI (#630); 64-bit funcs keep multi-GB caches whole on Windows LLP64. */
|
||||
static voidpf ZCALLBACK hts_zip_fopen_utf8(voidpf opaque, const void *filename,
|
||||
int mode) {
|
||||
const char *mode_fopen = NULL;
|
||||
|
||||
(void) opaque;
|
||||
if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER) == ZLIB_FILEFUNC_MODE_READ)
|
||||
mode_fopen = "rb";
|
||||
else if (mode & ZLIB_FILEFUNC_MODE_EXISTING)
|
||||
mode_fopen = "r+b";
|
||||
else if (mode & ZLIB_FILEFUNC_MODE_CREATE)
|
||||
mode_fopen = "wb";
|
||||
if (filename == NULL || mode_fopen == NULL)
|
||||
return NULL;
|
||||
return (voidpf) FOPEN((const char *) filename, mode_fopen);
|
||||
}
|
||||
|
||||
static unzFile hts_unzOpen_utf8(const char *path) {
|
||||
zlib_filefunc64_def ff;
|
||||
|
||||
fill_fopen64_filefunc(&ff);
|
||||
ff.zopen64_file = hts_zip_fopen_utf8;
|
||||
return unzOpen2_64(path, &ff);
|
||||
}
|
||||
|
||||
static zipFile hts_zipOpen_utf8(const char *path, int append) {
|
||||
zlib_filefunc64_def ff;
|
||||
|
||||
fill_fopen64_filefunc(&ff);
|
||||
ff.zopen64_file = hts_zip_fopen_utf8;
|
||||
return zipOpen2_64(path, append, NULL, &ff);
|
||||
}
|
||||
|
||||
/* Pathname of a file inside the mirror dir (rotating concat buffer). */
|
||||
@@ -874,9 +908,9 @@ static char *reconcile_path(httrackp *opt, const char *name) {
|
||||
/* Replace the new-generation file by the old one, when the old one exists. */
|
||||
static void reconcile_promote(httrackp *opt, const char *oldname,
|
||||
const char *newname) {
|
||||
if (fexist(reconcile_path(opt, oldname))) {
|
||||
remove(reconcile_path(opt, newname));
|
||||
rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
|
||||
if (fexist_utf8(reconcile_path(opt, oldname))) {
|
||||
UNLINK(reconcile_path(opt, newname));
|
||||
RENAME(reconcile_path(opt, oldname), reconcile_path(opt, newname));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -885,7 +919,7 @@ void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
|
||||
case CACHE_RECONCILE_PROMOTE:
|
||||
/* Previous run rotated new.* to old.* then died before writing: promote
|
||||
the old generation back, whichever format it uses. */
|
||||
if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
if (!fexist_utf8(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
break;
|
||||
case CACHE_RECONCILE_INTERRUPTED:
|
||||
@@ -894,16 +928,17 @@ void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
|
||||
is -1 for a missing file, which would spuriously pass the "< TINY" test
|
||||
and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
|
||||
*/
|
||||
if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
|
||||
if (!opt->cache ||
|
||||
!fexist_utf8(reconcile_path(opt, "hts-in_progress.lock")))
|
||||
break;
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")) <
|
||||
if (fexist_utf8(reconcile_path(opt, "hts-cache/new.zip")) &&
|
||||
fexist_utf8(reconcile_path(opt, "hts-cache/old.zip")) &&
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/new.zip")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
break;
|
||||
case CACHE_RECONCILE_ROLLBACK:
|
||||
@@ -940,24 +975,26 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
#endif
|
||||
if (!cache->ro) {
|
||||
#ifdef _WIN32
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"));
|
||||
/* Windows mkdir takes no mode; use the UTF-8 wrapper for #630. */
|
||||
MKDIR(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache"));
|
||||
#else
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
|
||||
/* keep the cache dir 0700, not MKDIR's 0755. */
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache"),
|
||||
HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
if ((fexist(fconcat(
|
||||
if ((fexist_utf8(fconcat(
|
||||
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))) { // a previous cache exists.. rename it
|
||||
/* Remove OLD cache */
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (remove
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) != 0) {
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")) !=
|
||||
0) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"Cache: error while moving previous cache");
|
||||
}
|
||||
@@ -980,33 +1017,27 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: no cache found");
|
||||
}
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: size %d",
|
||||
(int)
|
||||
fsize(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")));
|
||||
(int) fsize_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")));
|
||||
|
||||
// charger index cache précédent
|
||||
if ((!cache->ro
|
||||
&&
|
||||
fsize(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) > 0)
|
||||
|| (cache->ro
|
||||
&&
|
||||
fsize(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")) > 0)
|
||||
) {
|
||||
if ((!cache->ro &&
|
||||
fsize_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")) >
|
||||
0) ||
|
||||
(cache->ro &&
|
||||
fsize_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip")) >
|
||||
0)) {
|
||||
if (!cache->ro) {
|
||||
cache->zipInput =
|
||||
unzOpen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"));
|
||||
cache->zipInput = hts_unzOpen_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip"));
|
||||
} else {
|
||||
cache->zipInput =
|
||||
unzOpen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
cache->zipInput = hts_unzOpen_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"));
|
||||
}
|
||||
|
||||
// Corrupted ZIP file ? Try to repair!
|
||||
@@ -1026,6 +1057,8 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
}
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Cache: damaged cache, trying to repair");
|
||||
/* mztools has no UTF-8 hook, so repairing a corrupt cache under a
|
||||
non-ASCII path_log fails cleanly (re-crawl), never forks a twin. */
|
||||
if (unzRepair
|
||||
(name,
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
@@ -1033,11 +1066,11 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/repair.tmp"),
|
||||
&repaired, &repairedBytes) == Z_OK) {
|
||||
unlink(name);
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/repair.zip"), name);
|
||||
cache->zipInput = unzOpen(name);
|
||||
UNLINK(name);
|
||||
RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/repair.zip"),
|
||||
name);
|
||||
cache->zipInput = hts_unzOpen_utf8(name);
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Cache: %d bytes successfully recovered in %d entries",
|
||||
(int) repairedBytes, (int) repaired);
|
||||
@@ -1133,8 +1166,8 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"Cache: error trying to open the cache");
|
||||
}
|
||||
|
||||
} else if (fsize(reconcile_path(opt, "hts-cache/old.ndx")) > 0 ||
|
||||
fsize(reconcile_path(opt, "hts-cache/new.ndx")) > 0) {
|
||||
} else if (fsize_utf8(reconcile_path(opt, "hts-cache/old.ndx")) > 0 ||
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/new.ndx")) > 0) {
|
||||
/* pre-3.31 (2003) .dat/.ndx cache: import support removed */
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Cache: the pre-3.31 .dat/.ndx cache format is no longer "
|
||||
@@ -1150,67 +1183,58 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
#endif
|
||||
if (!cache->ro) {
|
||||
// ouvrir caches actuels
|
||||
structcheck(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/"));
|
||||
structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/"));
|
||||
|
||||
{
|
||||
/* Create ZIP file cache */
|
||||
cache->zipOutput =
|
||||
(void *)
|
||||
zipOpen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"), 0);
|
||||
cache->zipOutput = (void *) hts_zipOpen_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"),
|
||||
0);
|
||||
|
||||
if (cache->zipOutput != NULL) {
|
||||
// supprimer old.lst
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.lst"));
|
||||
// renommer
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst")))
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst")))
|
||||
RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst"),
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.lst"));
|
||||
// ouvrir
|
||||
cache->lst =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"), "wb");
|
||||
FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst"),
|
||||
"wb");
|
||||
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
|
||||
opt->state.strc.lst = cache->lst;
|
||||
|
||||
// supprimer old.txt
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.txt"));
|
||||
// renommer
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt")))
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt"));
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt")))
|
||||
RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.txt"),
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.txt"));
|
||||
// ouvrir
|
||||
cache->txt =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"), "wb");
|
||||
FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.txt"),
|
||||
"wb");
|
||||
if (cache->txt) {
|
||||
fprintf(cache->txt,
|
||||
"date\tsize'/'remotesize\tflags(request:Update,Range state:File response:Modified,Chunked,gZipped)\t");
|
||||
|
||||
@@ -3293,10 +3293,10 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (ok) {
|
||||
if (!back_exist
|
||||
(sback, opt, heap(p)->adr, heap(p)->fil, heap(p)->sav)) {
|
||||
if (back_add
|
||||
(sback, opt, cache, heap(p)->adr, heap(p)->fil, heap(p)->sav,
|
||||
heap(heap(p)->precedent)->adr, heap(heap(p)->precedent)->fil,
|
||||
heap(p)->testmode) == -1) {
|
||||
if (back_add(sback, opt, cache, heap(p)->adr, heap(p)->fil,
|
||||
heap(p)->sav, heap(heap(p)->precedent)->adr,
|
||||
heap(heap(p)->precedent)->fil, heap(p)->testmode,
|
||||
heap(p)->refetch_whole) == -1) {
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"error: unable to add more links through back_add for back_fill");
|
||||
#if BDEBUG==1
|
||||
|
||||
@@ -77,17 +77,17 @@ void infomsg(const char *msg) {
|
||||
if (msg[2] != ' ') {
|
||||
if ((msg[3] == ' ') || (msg[4] == ' ')) {
|
||||
char cmd[32] = "-";
|
||||
int p = 0;
|
||||
int p;
|
||||
|
||||
while(cmd[p] == ' ')
|
||||
p++;
|
||||
sscanf(msg + p, "%s", cmd + strlen(cmd));
|
||||
/* clears cN -> c */
|
||||
if ((p = (int) strlen(cmd)) > 2)
|
||||
if (cmd[p - 1] == 'N')
|
||||
cmd[p - 1] = '\0';
|
||||
/* finds alias (if any) */
|
||||
sscanf(msg, "%30s", cmd + strlen(cmd));
|
||||
/* try the flag as-is, then strip a trailing N as the numeric-arg
|
||||
placeholder (cN -> c); this order keeps -%N from becoming -% */
|
||||
p = optreal_find(cmd);
|
||||
if (p < 0 && (int) strlen(cmd) > 2 &&
|
||||
cmd[strlen(cmd) - 1] == 'N') {
|
||||
cmd[strlen(cmd) - 1] = '\0';
|
||||
p = optreal_find(cmd);
|
||||
}
|
||||
if (p >= 0) {
|
||||
/* fings type of parameter: number,param,param concatenated,single cmd */
|
||||
if (strcmp(opttype_value(p), "param") == 0)
|
||||
@@ -648,9 +648,8 @@ void help(const char *app, int more) {
|
||||
infomsg("Command-line specific options:");
|
||||
infomsg
|
||||
(" V execute system command after each files ($0 is the filename: -V \"rm \\$0\")");
|
||||
infomsg
|
||||
(" %W use an external library function as a wrapper (-%W myfoo.so[,myparameters])");
|
||||
/* infomsg(" %O do a chroot before setuid"); */
|
||||
infomsg(" %W use an external library function as a wrapper (-%W "
|
||||
"myfoo.so[,myparameters])");
|
||||
infomsg("");
|
||||
infomsg("Details: Option N");
|
||||
infomsg(" N0 Site-structure (default)");
|
||||
|
||||
@@ -615,9 +615,9 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
strcpybuff(current.fil, fil_complete);
|
||||
// ajouter dans le backing le fichier en mode test
|
||||
// savename: rien car en mode test
|
||||
if (back_add
|
||||
(sback, opt, cache, current.adr, current.fil, BACK_ADD_TEST,
|
||||
referer_adr, referer_fil, 1) != -1) {
|
||||
if (back_add(sback, opt, cache, current.adr, current.fil,
|
||||
BACK_ADD_TEST, referer_adr, referer_fil, 1,
|
||||
HTS_FALSE) != -1) {
|
||||
int b;
|
||||
|
||||
b = back_index(opt, sback, current.adr, current.fil, BACK_ADD_TEST);
|
||||
@@ -706,7 +706,10 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (!hts_wait_available_socket(sback, opt,
|
||||
cache, ptr))
|
||||
return -1;
|
||||
if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) { // OK
|
||||
if (back_add(sback, opt, cache, moved.adr,
|
||||
moved.fil, methode, referer_adr,
|
||||
referer_fil, 1,
|
||||
HTS_FALSE) != -1) { // OK
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"(during prefetch) %s (%d) to link %s at %s%s",
|
||||
back[b].r.msg,
|
||||
@@ -725,7 +728,8 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
has_been_moved = 1; // sinon ne pas forcer has_been_moved car non déplacé
|
||||
petits_tours++;
|
||||
//
|
||||
} else { // sinon on fait rien et on s'en va.. (ftp etc)
|
||||
} else { // sinon on fait rien et on s'en va..
|
||||
// (ftp etc)
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"Warning: Savename redirect backing error at %s%s",
|
||||
moved.adr, moved.fil);
|
||||
@@ -796,7 +800,6 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Unexpected savename backing error at %s%s", adr,
|
||||
fil_complete);
|
||||
|
||||
}
|
||||
// restaurer
|
||||
opt->state._hts_in_html_parsing = hihp;
|
||||
|
||||
@@ -658,6 +658,9 @@ struct htsblk {
|
||||
int debugid; /**< connection debug id */
|
||||
/* */
|
||||
htsrequest req; /**< parameters used for the request */
|
||||
/* Restart-whole signal: a resume this response rejected (unusable 206) must
|
||||
retry with no Range, else a surviving partial/temp-ref loops (#581). */
|
||||
hts_boolean refetch_wholefile;
|
||||
/*char digest[32+2]; // md5 digest generated by the engine ("" if none) */
|
||||
};
|
||||
|
||||
@@ -681,6 +684,9 @@ struct lien_url {
|
||||
char link_import; /**< imported after a move; skip the usual up/down rules */
|
||||
int retry; /**< remaining retries */
|
||||
int testmode; /**< test only: send just a HEAD */
|
||||
hts_boolean
|
||||
refetch_whole; /**< force a whole-file GET, ignoring any partial/temp-ref
|
||||
resume, so a rejected 206 can't loop (#581) */
|
||||
};
|
||||
|
||||
/* A file being fetched in the background. */
|
||||
|
||||
@@ -315,9 +315,18 @@ static void escape_url_parens(char *const s, const size_t size) {
|
||||
strlcpybuff(s, buff, size);
|
||||
}
|
||||
|
||||
/* Strip a default ":80" from lien's authority in place. Any spelling that
|
||||
range-parses to 80 (":80", ":080") is dropped by its matched length, not a
|
||||
hardcoded 3 chars; a value that merely wraps to 80 as an int (#614) stays. */
|
||||
/* Default port for lien's scheme (case-insensitive), 80 if absent/unknown; so
|
||||
schemeless and protocol-relative //host links default to 80 (known gap). */
|
||||
static int scheme_default_port(const char *lien) {
|
||||
if (strfield(lien, "https:"))
|
||||
return 443;
|
||||
if (strfield(lien, "ftp:"))
|
||||
return 21;
|
||||
return 80;
|
||||
}
|
||||
|
||||
/* Strip the scheme's own default port (80 http, 443 https, 21 ftp) from lien's
|
||||
authority in place; :80 on https/ftp stays as a real port (#638, #614). */
|
||||
void hts_strip_default_port(char *lien, size_t size) {
|
||||
char *a;
|
||||
|
||||
@@ -339,7 +348,8 @@ void hts_strip_default_port(char *lien, size_t size) {
|
||||
b++;
|
||||
saved = *b;
|
||||
*b = '\0';
|
||||
is_default = hts_parse_url_port(a + 1, &port) && port == 80;
|
||||
is_default =
|
||||
hts_parse_url_port(a + 1, &port) && port == scheme_default_port(lien);
|
||||
*b = saved;
|
||||
if (is_default) { // default port, strip it
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
@@ -3746,6 +3756,9 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
heap_top()->retry = heap(ptr)->retry - 1; // moins 1 retry!
|
||||
heap_top()->premier = heap(ptr)->premier;
|
||||
heap_top()->precedent = heap(ptr)->precedent;
|
||||
// a rejected resume (unusable 206) must refetch whole, no Range
|
||||
// (#581)
|
||||
heap_top()->refetch_whole = r->refetch_wholefile;
|
||||
} else { // oups erreur, plus de mémoire!!
|
||||
return 0;
|
||||
}
|
||||
@@ -3977,18 +3990,17 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
|
||||
#if BDEBUG==1
|
||||
printf("crash backing: %s%s\n", heap(ptr)->adr, heap(ptr)->fil);
|
||||
#endif
|
||||
if (back_add
|
||||
(sback, opt, cache, urladr(), urlfil(), savename(),
|
||||
heap(heap(ptr)->precedent)->adr, heap(heap(ptr)->precedent)->fil,
|
||||
heap(ptr)->testmode) == -1) {
|
||||
if (back_add(sback, opt, cache, urladr(), urlfil(), savename(),
|
||||
heap(heap(ptr)->precedent)->adr,
|
||||
heap(heap(ptr)->precedent)->fil, heap(ptr)->testmode,
|
||||
heap(ptr)->refetch_whole) == -1) {
|
||||
printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",
|
||||
__LINE__);
|
||||
#if BDEBUG==1
|
||||
printf("error while crash adding\n");
|
||||
#endif
|
||||
hts_log_print(opt, LOG_ERROR, "Unexpected backing error for %s%s", urladr(),
|
||||
urlfil());
|
||||
|
||||
hts_log_print(opt, LOG_ERROR, "Unexpected backing error for %s%s",
|
||||
urladr(), urlfil());
|
||||
}
|
||||
}
|
||||
#if BDEBUG==1
|
||||
|
||||
@@ -1589,10 +1589,8 @@ static int st_identabs(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Default-port strip (#627): a genuine 80 (any spelling) is removed by its
|
||||
matched length, host preserved; a non-80 port or one that only wraps to 80 as
|
||||
a 32-bit int (#614) is left intact. Guards the old bug where ":080"/":0080"
|
||||
dropped a hardcoded 3 chars and glued the leftover digits onto the host. */
|
||||
/* Default-port strip is scheme-aware (#638), overflow-safe (#614): a scheme's
|
||||
own default (any spelling) is dropped, a real port stays; guards #627. */
|
||||
static int st_stripport(httrackp *opt, int argc, char **argv) {
|
||||
static const struct {
|
||||
const char *in, *out;
|
||||
@@ -1606,6 +1604,13 @@ static int st_stripport(httrackp *opt, int argc, char **argv) {
|
||||
{"http://127.0.0.1:8080/x", "http://127.0.0.1:8080/x"},
|
||||
{"http://127.0.0.1:4294967376/x", "http://127.0.0.1:4294967376/x"},
|
||||
{"http://127.0.0.1/x", "http://127.0.0.1/x"},
|
||||
{"https://127.0.0.1:443/x", "https://127.0.0.1/x"},
|
||||
{"https://127.0.0.1:80/x", "https://127.0.0.1:80/x"},
|
||||
// Scheme match is case-insensitive: HTTPS' default is 443, so :80 stays.
|
||||
{"HTTPS://127.0.0.1:80/x", "HTTPS://127.0.0.1:80/x"},
|
||||
{"ftp://127.0.0.1:21/x", "ftp://127.0.0.1/x"},
|
||||
{"ftp://127.0.0.1:80/x", "ftp://127.0.0.1:80/x"},
|
||||
{"http://127.0.0.1:443/x", "http://127.0.0.1:443/x"},
|
||||
};
|
||||
|
||||
size_t k;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
--- mztools.c.orig 2024-01-27 14:07:18.636193212 +0100
|
||||
+++ mztools.c 2024-01-27 14:09:55.356620093 +0100
|
||||
@@ -10,6 +10,7 @@
|
||||
@@ -10,10 +10,11 @@
|
||||
#include <string.h>
|
||||
#include "zlib.h"
|
||||
#include "unzip.h"
|
||||
@@ -8,6 +8,11 @@
|
||||
|
||||
#define READ_8(adr) ((unsigned char)*(adr))
|
||||
#define READ_16(adr) ( READ_8(adr) | (READ_8(adr+1) << 8) )
|
||||
-#define READ_32(adr) ( READ_16(adr) | (READ_16((adr)+2) << 16) )
|
||||
+#define READ_32(adr) ((uLong) READ_16(adr) | ((uLong) READ_16((adr) + 2) << 16))
|
||||
|
||||
#define WRITE_8(buff, n) do { \
|
||||
*((unsigned char*)(buff)) = (unsigned char) ((n) & 0xff); \
|
||||
@@ -141,8 +142,8 @@
|
||||
/* Central directory entry */
|
||||
{
|
||||
|
||||
@@ -6,6 +6,7 @@ set -euo pipefail
|
||||
# Stripping a default :80 from a crawled link used to skip a hardcoded 3 chars
|
||||
# (#627): ":080"/":0080" lost only ":80" and glued the rest onto the host
|
||||
# (127.0.0.1:080/x -> 127.0.0.10/x), and a value wrapping to 80 as a 32-bit int
|
||||
# (#614) was stripped as if it were the default. All assertions live in the
|
||||
# engine self-test.
|
||||
# (#614) was stripped as if it were the default. The default is scheme-aware
|
||||
# (#638): an explicit :80 on https/ftp stays, :443/:21 strip under their own
|
||||
# scheme. All assertions live in the engine self-test.
|
||||
httrack -O /dev/null -#test=stripport | grep -q "stripport self-test OK"
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The
|
||||
# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin:
|
||||
# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage.
|
||||
# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so
|
||||
# a log written to the twin fails them. POSIX has no twin, so this bites on the
|
||||
# Windows CI leg (like test 64). The cache twin is a separate, larger fix.
|
||||
# logs (hts-log.txt/hts-err.txt) and the hts-cache must land there, not in an
|
||||
# ANSI-mangled twin: on Windows path_log holds UTF-8 bytes the raw file calls
|
||||
# read as the codepage. The --log-found audit greps logroot=café/hts-log.txt and
|
||||
# --cache-under-logroot checks café/hts-cache/new.zip, so anything written to the
|
||||
# twin fails them. POSIX has no twin, so this bites on the Windows CI leg (test
|
||||
# 64), exercising only the code path elsewhere.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \
|
||||
--found 'simple/basic.html' \
|
||||
--log-found 'mirror complete in' \
|
||||
--cache-under-logroot \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
|
||||
@@ -16,11 +16,12 @@ root=$(nativepath "${testdir}/server-root")
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
# On Windows the pass-1 interrupt is a hard kill (MSYS can't signal a native
|
||||
# exe) and the restart-whole path fails on the repaired cache (#581) -- the very
|
||||
# bug this exercises; skip until the engine fix lands.
|
||||
# Windows-only repro of #581 (a hard-killed pass 1 + repaired cache loses the
|
||||
# file). The engine now forces a no-Range whole refetch on the restart, but that
|
||||
# fix is unverified on Windows CI, and this test's pass-1 interrupt/port-race
|
||||
# behavior there is unconfirmed; lift this skip on a Windows runner to verify.
|
||||
if is_windows; then
|
||||
echo "Windows: restart-whole fails on a repaired cache (#581), skipping"
|
||||
echo "Windows: #581 fix unverified on CI, skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
|
||||
@@ -118,6 +118,9 @@ while test "$pos" -lt "$nargs"; do
|
||||
nopurge=1
|
||||
audit+=("--no-purge")
|
||||
;;
|
||||
--cache-under-logroot)
|
||||
audit+=("--cache-under-logroot")
|
||||
;;
|
||||
--tls)
|
||||
tls=1
|
||||
scheme=https
|
||||
@@ -379,6 +382,15 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
|
||||
assert_equals "checking files" "${audit[$i]}" "$nFiles"
|
||||
;;
|
||||
--cache-under-logroot)
|
||||
# The cache must sit under path_log (logroot), not an ANSI-mangled twin
|
||||
# of a non-ASCII -O dir (#630 cache half). Bites only on the Windows leg.
|
||||
info "checking cache under logroot"
|
||||
if test -e "${logroot}/hts-cache/new.zip"; then result "OK"; else
|
||||
result "cache not under logroot (mangled twin?)"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--found)
|
||||
i=$((i + 1))
|
||||
info "checking for ${audit[$i]}"
|
||||
|
||||
Reference in New Issue
Block a user