mirror of
https://github.com/xroche/httrack.git
synced 2026-07-22 16:51:25 +03:00
Compare commits
2 Commits
master
...
worktree-a
| Author | SHA1 | Date | |
|---|---|---|---|
|
b6f794b404 | ||
|
|
c791e73ef7 |
28
.github/workflows/ci.yml
vendored
28
.github/workflows/ci.yml
vendored
@@ -618,31 +618,3 @@ jobs:
|
||||
echo "Fix locally with: git clang-format --binary clang-format-19 $base"
|
||||
exit 1 ;;
|
||||
esac
|
||||
|
||||
man-page-sync:
|
||||
name: man page / html in sync
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# html/httrack.man.html is groff-rendered from man/httrack.1 and committed.
|
||||
# Rendering needs the full groff html device, so CI can't regenerate it;
|
||||
# instead require the two to move together: a PR that touches httrack.1
|
||||
# must also touch the html, catching the "regenerated roff, forgot html".
|
||||
- name: httrack.1 changes must include html/httrack.man.html
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin \
|
||||
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
|
||||
base="origin/${{ github.base_ref }}"
|
||||
changed="$(git diff --name-only "$base"...HEAD)"
|
||||
has() { printf '%s\n' "$changed" | grep -qx "$1"; }
|
||||
if has man/httrack.1 && ! has html/httrack.man.html; then
|
||||
echo "::error::man/httrack.1 changed but html/httrack.man.html did not."
|
||||
echo "Regenerate it with: make -C man regen-man-html (needs the full groff package)."
|
||||
exit 1
|
||||
fi
|
||||
echo "man/html sync OK."
|
||||
|
||||
3
.github/workflows/codeql.yml
vendored
3
.github/workflows/codeql.yml
vendored
@@ -55,9 +55,6 @@ jobs:
|
||||
query-filters:
|
||||
- exclude:
|
||||
id: cpp/world-writable-file-creation
|
||||
# Models auth-bypass-by-spoofing; httrack has no auth surface, its +/- crawl filter is a mirror boundary, not a security one.
|
||||
- exclude:
|
||||
id: cpp/user-controlled-bypass
|
||||
|
||||
# Manual build: CodeQL traces the compiler, so build exactly what ships.
|
||||
- name: Build
|
||||
|
||||
@@ -217,7 +217,7 @@ school or
|
||||
shows. They might do that because they are connected through expensive modem connection,
|
||||
or because they would like to consult pages while travelling, or archive sites that may be
|
||||
removed
|
||||
one day, make some data mining, compiling information ("if only I could find this
|
||||
one day, make some data mining, comiling information ("if only I could find this
|
||||
website I saw one day.."). <br>
|
||||
There are many good reasons to mirror websites, and this helps many good people.<br>
|
||||
As a webmaster, you might be interested to use such tools, too: test broken links, move a
|
||||
@@ -229,7 +229,7 @@ test the webserver response and performances, index it..<br>
|
||||
Anyway, bandwidth abuse can be a problem. If your site is regularly "clobbered"
|
||||
by evil downloaders, you have <br>
|
||||
various solutions. You have radical solutions, and intermediate solutions. I strongly
|
||||
recommend not to use<br>
|
||||
recomment not to use<br>
|
||||
radical solutions, because of the previous remarks (good people often mirror websites).<br>
|
||||
<br>
|
||||
In general, for all solutions,<br>
|
||||
@@ -244,7 +244,7 @@ or, to be extreme: if you unplug the wire, there will be no bandwidth abuse<br>
|
||||
Good: Will work with good people. Many good people just don't KNOW that they can slow down
|
||||
a network.<br>
|
||||
Bad: Will **only** work with good people<br>
|
||||
How to do: Obvious - place a note, a warning, an article, a draw, a poem or whatever you
|
||||
How to do: Obvious - place a note, a warning, an article, a draw, a poeme or whatever you
|
||||
want<br>
|
||||
<br>
|
||||
</li><li>Use "robots.txt" file<br>
|
||||
@@ -266,7 +266,7 @@ Good: Efficient<br>
|
||||
Bad: Multiple users behind proxies will be slow down, not really easy to setup<br>
|
||||
How to do: Depends on webserver. Might be done with low-level IP rules (QoS)<br>
|
||||
<br>
|
||||
</li><li>Prioritize small files, against large files<br>
|
||||
</li><li>Priorize small files, against large files<br>
|
||||
Good: Efficient if large files are the cause of abuse<br>
|
||||
Bad: Not always efficient<br>
|
||||
How to do: Depends on the webserver<br>
|
||||
@@ -283,7 +283,7 @@ How to do: Use routine QoS (fair queuing), or webserver options<br>
|
||||
<br>
|
||||
</li><li>Use technical tricks (like javascript) to hide URLs<br>
|
||||
Good: Efficient<br>
|
||||
Bad: The most efficient tricks will also cause your website to be heavy, and not
|
||||
Bad: The most efficient tricks will also cause your website to he heavy, and not
|
||||
user-friendly (and therefore less attractive, even for surfing users). Remember: clients
|
||||
or visitors might want to consult offline your website. Advanced users will also be still
|
||||
able to note the URLs and catch them. Will not work on non-javascript browsers. It will
|
||||
@@ -335,7 +335,7 @@ Example: Use things like
|
||||
</li><li>Use technical tricks to temporarily ban IPs<br>
|
||||
Good: Efficient<br>
|
||||
Bad: Radical (your site will only be available online for all users), not easy to setup<br>
|
||||
How to do: Create fake links with "killing" targets<br>
|
||||
How to to: Create fake links with "killing" targets<br>
|
||||
Example: Use things like <a href="killme.cgi"><nothing></a>
|
||||
(again an example in php:)<br>
|
||||
<tt>
|
||||
@@ -417,7 +417,7 @@ Example:<br>
|
||||
</li><li>Another one is to create images of emails<br>
|
||||
Good: Efficient, does not require javascript<br>
|
||||
Bad: There is still the problem of the link (mailto:), images are bigger than text, and it can cause problems for blind people (a good solution is use an ALT attribute with the email written like "smith at mycompany dot com")<br>
|
||||
How to do: Not so obvious if you do not want to create images by yourself<br>
|
||||
How to do: Not so obvious of you do not want to create images by yourself<br>
|
||||
Example: (php, Unix)<br>
|
||||
|
||||
<tt>
|
||||
@@ -491,7 +491,7 @@ echo <br>
|
||||
</li><li>You can also create temporary email aliases, each week, for all users<br>
|
||||
Good: Efficient, and you can give your real email in your reply-to address<br>
|
||||
Bad: Temporary emails<br>
|
||||
How to do: Not so hard to do<br>
|
||||
How to do: Not so hard todo<br>
|
||||
Example: (script & php, Unix)<br>
|
||||
|
||||
<tt>
|
||||
|
||||
157
html/cmddoc.html
Normal file
157
html/cmddoc.html
Normal file
@@ -0,0 +1,157 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>Command-Line Documentation</em></h2>
|
||||
|
||||
<br>
|
||||
|
||||
The command-line version
|
||||
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
<br>A task-oriented guide: how to do the common things, and the defaults that surprise newcomers<br><br>
|
||||
<li><a href="options.html">Command line Options</a></li>
|
||||
<br>List of all powerful command line options<br><br>
|
||||
<li>How to use httrack command-line version:</li>
|
||||
<ul>
|
||||
<li>Open a shell window</li>
|
||||
<br>
|
||||
<br>
|
||||
<li>Type in <tt>httrack</tt> (or the complete path to the httrack executable)</li>
|
||||
<br><small><tt>httrack</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Add the URLs, separated by a blank space</li>
|
||||
<br><small><tt>httrack www.example.com/foo/</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>If you need, add some options (see the <a href="options.html">option list</a>)</li>
|
||||
<br><small><tt>httrack www.example.com/foo/ -O "/webs" -N4 -P proxy.myhost.com:3128</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Launch the command line, and wait until the mirror is finishing</li>
|
||||
<br><small>You can (especially on the Unix release) press ^C to stop the mirror or put httrack in background</small>
|
||||
<br>
|
||||
<br>
|
||||
</ul>
|
||||
</ul>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
@@ -219,23 +219,6 @@ that a <tt>403 Forbidden</tt> is a server refusal, not a robots rule: robots
|
||||
options will not help there. That is an
|
||||
<a href="#identity">identity</a> problem.</p>
|
||||
|
||||
<h4>Filter wildcards</h4>
|
||||
<p>Inside a filter pattern, <tt>*</tt> matches any run of characters; a few
|
||||
bracket forms match narrower sets. The full table, with size and mime rules, is on
|
||||
<a href="filters.html">the filters page</a>.</p>
|
||||
<table class="tblRegular tableWidth" border="0">
|
||||
<tr class="tblHeaderColor"><td><b>Wildcard</b></td><td><b>Matches</b></td><td><b>Example</b></td></tr>
|
||||
<tr><td><tt>*</tt></td><td>any run of characters</td><td><tt>+*.pdf</tt> — any URL ending <tt>.pdf</tt></td></tr>
|
||||
<tr><td><tt>*[file]</tt>, <tt>*[name]</tt></td><td>one path segment (any char but <tt>/</tt> and <tt>?</tt>)</td><td><tt>example.com/*[file]/</tt> — a directory-index page</td></tr>
|
||||
<tr><td><tt>*[path]</tt></td><td>a path, slashes allowed (any char but <tt>?</tt>)</td><td><tt>example.com/*[path].zip</tt></td></tr>
|
||||
<tr><td><tt>*[param]</tt></td><td>an optional query string</td><td><tt>page.html*[param]</tt> matches with or without <tt>?...</tt></td></tr>
|
||||
<tr><td><tt>*[a,b,c]</tt></td><td>any one character in the set</td><td><tt>*[a,b,c].txt</tt></td></tr>
|
||||
<tr><td><tt>*[a-z]</tt></td><td>any one character in the range</td><td><tt>img*[0-9].gif</tt></td></tr>
|
||||
<tr><td><tt>*[\x]</tt></td><td>the literal character x (escapes <tt>* [ ] \</tt>)</td><td><tt>*[\*]</tt> matches a real <tt>*</tt></td></tr>
|
||||
<tr><td><tt>*[<NN]</tt>, <tt>*[>NN]</tt></td><td>file size in KB below / above NN</td><td><tt>-*.gif*[<5]</tt> skips GIFs under 5 KB</td></tr>
|
||||
<tr><td><tt>*[]</tt></td><td>end anchor: nothing may follow</td><td><tt>*.html*[]</tt> rejects <tt>i.html?p=1</tt></td></tr>
|
||||
</table>
|
||||
|
||||
<h3 id="limits">4. Limits and politeness</h3>
|
||||
|
||||
<p>HTTrack ships cautious on purpose: it is easy to hammer a small site by
|
||||
@@ -251,7 +234,7 @@ below let you go faster when you own the target, and slower when you do not.</p>
|
||||
<tr><td><tt>--max-time (-E)</tt></td><td>Stop after N seconds of wall-clock time.</td></tr>
|
||||
<tr><td><tt>--max-files (-m)</tt></td><td>Per-file size caps.</td></tr>
|
||||
<tr><td><tt>--timeout (-T), --retries (-R), --min-rate (-J), --host-control (-H)</tt></td><td>Idle timeout, retry count, minimum acceptable rate, and host-ban behavior for slow or dead hosts.</td></tr>
|
||||
<tr><td><tt>--max-pause (-G), --pause (-%G)</tt></td><td>Pause the mirror at N bytes, or pause between files, to spread the load.</td></tr>
|
||||
<tr><td><tt>--max-pause (-G), -%G</tt></td><td>Pause the mirror at N bytes, or pause between files, to spread the load.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>The security clamps.</b> To keep an accidental typo from turning into a flood,
|
||||
@@ -274,8 +257,8 @@ really HTML.</p>
|
||||
<tr><td><tt>--structure (-N)</tt></td><td>The local path and name layout. Presets are numeric, and you can also give a template such as <tt>--structure "%h%p/%n%q.%t"</tt>.</td></tr>
|
||||
<tr><td><tt>--long-names (-L)</tt></td><td>Long names, 8.3 names, or ISO9660 for CD masters.</td></tr>
|
||||
<tr><td><tt>--assume (-%A)</tt></td><td>Assume a MIME type for an extension, for example <tt>--assume php=text/html</tt>. This also skips the extra HEAD probe HTTrack would otherwise send to learn the type.</td></tr>
|
||||
<tr><td><tt>--delayed-type-check (-%N), --cached-delayed-type-check (-%D), --check-type (-u), -%t</tt></td><td>When and how the content type is checked, and whether the original extension is kept.</td></tr>
|
||||
<tr><td><tt>--include-query-string (-%q), --strip-query (-%g)</tt></td><td>Whether the query string appears in the local filename, and whether query keys are stripped when deciding if two URLs are the same file.</td></tr>
|
||||
<tr><td><tt>-%N, --cached-delayed-type-check (-%D), --check-type (-u), -%t</tt></td><td>When and how the content type is checked, and whether the original extension is kept.</td></tr>
|
||||
<tr><td><tt>--include-query-string (-%q), -%g</tt></td><td>Whether the query string appears in the local filename, and whether query keys are stripped when deciding if two URLs are the same file.</td></tr>
|
||||
</table>
|
||||
|
||||
<p>The <tt>-N</tt> presets are built from modular arithmetic on the name fields, so
|
||||
@@ -346,7 +329,7 @@ across links at the same time.</p>
|
||||
<tr><td><tt>--proxy (-P)</tt></td><td>Route through a proxy. HTTP, SOCKS5 and CONNECT are supported: <tt>-P host:8080</tt>, <tt>-P socks5://host:1080</tt>, <tt>-P connect://host:443</tt>, with optional <tt>user:pass@</tt>.</td></tr>
|
||||
<tr><td><tt>--httpproxy-ftp (-%f)</tt></td><td>Send FTP requests through the HTTP proxy.</td></tr>
|
||||
<tr><td><tt>--protocol (-@i)</tt></td><td>Prefer IPv4 or IPv6.</td></tr>
|
||||
<tr><td><tt>--http-10 (-%h), --keep-alive (-%k), --disable-compression (-%z)</tt></td><td>Force HTTP/1.0 (drops keep-alive and compression, useful for fragile CGI), toggle keep-alive, and toggle compression.</td></tr>
|
||||
<tr><td><tt>--http-10 (-%h), --keep-alive (-%k), -%z</tt></td><td>Force HTTP/1.0 (drops keep-alive and compression, useful for fragile CGI), toggle keep-alive, and toggle compression.</td></tr>
|
||||
<tr><td><tt>--bind (-%b), --tolerant (-%B)</tt></td><td>Bind to a local address, and accept technically-bogus responses some servers send.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -370,7 +353,7 @@ away and you lose the ability to continue or update the mirror.</p>
|
||||
<tr><td><tt>--update</tt></td><td>Re-run the mirror, revalidating each page with the server (If-Modified-Since / If-None-Match) and downloading only what changed.</td></tr>
|
||||
<tr><td><tt>--purge-old=0 (-X0)</tt></td><td>Do not purge. By default an update deletes local files that are no longer part of the mirror; <tt>--purge-old=0</tt> keeps them.</td></tr>
|
||||
<tr><td><tt>--cache (-C)</tt></td><td>Cache mode. The default already does the right thing and switches to update-checking when it detects an existing mirror.</td></tr>
|
||||
<tr><td><tt>--urlhack (-%u), --keep-www-prefix (-%j), --keep-double-slashes (-%o), --keep-query-order (-%y), --do-not-recatch (-%n), --updatehack (-%s), --store-all-in-cache (-k)</tt></td><td>URL-deduplication behavior and cache storage details.</td></tr>
|
||||
<tr><td><tt>--urlhack (-%u), -%j, -%o, -%y, --do-not-recatch (-%n), --updatehack (-%s), --store-all-in-cache (-k)</tt></td><td>URL-deduplication behavior and cache storage details.</td></tr>
|
||||
<tr><td><tt>--debug-cache (-#C), --repair-cache (-#R), --clean</tt></td><td>Inspect the cache, repair its ZIP, and erase cache plus logs.</td></tr>
|
||||
</table>
|
||||
|
||||
@@ -421,16 +404,14 @@ host and stops the crawl; start from the final URL, or add
|
||||
rule wins.</small></p>
|
||||
|
||||
<h4>Download the PDFs on a site</h4>
|
||||
<p><tt>httrack https://example.com/ "-*" "+https://example.com/*.html" "+https://example.com/*[path]/" "+https://example.com/*.pdf" --path mydir</tt><br>
|
||||
<small>HTTrack finds PDFs by parsing the site's HTML, so a plain
|
||||
<tt>"-*" "+example.com/*.pdf"</tt> is wrong: it prunes the pages that carry the
|
||||
links and keeps only PDFs reachable from the front page. Instead admit the HTML as
|
||||
scaffolding (<tt>*.html</tt> and <tt>*[path]/</tt> for directory-index pages at any
|
||||
depth, e.g. <tt>docs/</tt> or <tt>a/b/deep/</tt>; <tt>*[file]/</tt> would stop at one
|
||||
level), keep the PDFs, and let <tt>-*</tt> drop everything else (images,
|
||||
archives, off-site assets). PDFs on another host (a CDN or docs subdomain) are not
|
||||
included by default; allow that host too, e.g. <tt>"+docs.example.com/*.pdf"</tt>,
|
||||
or widen to <tt>"+*.pdf"</tt> for PDFs anywhere.</small></p>
|
||||
<p><tt>httrack https://example.com/ --path mydir</tt><br>
|
||||
<small>There is no PDF-only crawl. HTTrack discovers PDF links by parsing the
|
||||
site's HTML pages, so a <tt>"-*" "+example.com/*.pdf"</tt> filter blocks the very
|
||||
pages that carry the links and grabs only the PDFs linked from the front page. Let
|
||||
it crawl the site: the HTML pages come along as the scaffolding, and every reachable
|
||||
PDF is saved with them. If some PDFs live on another host (a CDN or a docs
|
||||
subdomain), allow that host too, for example
|
||||
<tt>"+docs.example.com/*.pdf"</tt>.</small></p>
|
||||
|
||||
<h4>Keep page requisites, including off-host images</h4>
|
||||
<p><tt>httrack https://example.com/blog/ --near --path mydir</tt><br>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, linux, windows, macos, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows vista, windows seven, windows 8, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
@@ -135,7 +135,9 @@ clear language!
|
||||
|
||||
<li><a href="#QG2">Where can I find French/other languages documentation?</a><br></li>
|
||||
|
||||
<li><a href="#QG3">Which systems does HTTrack run on?</a><br></li>
|
||||
<li><a href="#QG3b">Is HTTrack working on Windows Vista/Windows Seven/Windows 8 ?</a><br></li>
|
||||
|
||||
<li><a href="#QG3">Is HTTrack working on Windows 95/98 ?</a><br></li>
|
||||
|
||||
<li><a href="#QG4">What's the difference between HTTrack, WinHTTrack and WebHTTrack?</a><br></li>
|
||||
|
||||
@@ -318,10 +320,10 @@ This can easily done by using filters: go to the Option panel, select the 'Scan
|
||||
<tt>+www.example.com/gallery/trees/*<br>
|
||||
+www.example.com/photos/*</tt><br>
|
||||
<br>
|
||||
This means "accept all links beginning with <tt>www.example.com/gallery/trees/</tt> and <tt>www.example.com/photos/</tt>"
|
||||
This means "accept all links begining with <tt>www.example.com/gallery/trees/</tt> and <tt>www.example.com/photos/</tt>"
|
||||
- the <tt>+</tt> means "accept" and the final <tt>*</tt> means "any character will match after the previous ones".
|
||||
Remember the <tt>*.doc</tt> or <tt>*.zip</tt> encountered when you want to select all files from a certain type on your computer:
|
||||
it is almost the same here, except the beginning "+"<br>
|
||||
it is almost the same here, except the begining "+"<br>
|
||||
<br>
|
||||
Now, we might want to exclude all links in <tt>www.example.com/gallery/trees/hugetrees/</tt>, because with the previous filter,
|
||||
we accepted too many files. Here again, you can add a filter rule to refuse these links. Modify the previous filters to:<br>
|
||||
@@ -329,8 +331,8 @@ we accepted too many files. Here again, you can add a filter rule to refuse thes
|
||||
+www.example.com/photos/*<br>
|
||||
-www.example.com/gallery/trees/hugetrees/*</tt><br>
|
||||
<br>
|
||||
You have noticed the <tt>-</tt> in the beginning of the third rule: this means "refuse links matching the rule"
|
||||
; and the rule is "any files beginning with <tt>www.example.com/gallery/trees/hugetrees/</tt><br>
|
||||
You have noticed the <tt>-</tt> in the begining of the third rule: this means "refuse links matching the rule"
|
||||
; and the rule is "any files begining with <tt>www.example.com/gallery/trees/hugetrees/</tt><br>
|
||||
|
||||
Voila! With these three rules, you have precisely defined what you wanted to capture.<br>
|
||||
<br>
|
||||
@@ -359,12 +361,12 @@ You can freely download it, without paying any fees, copy it to your friends, an
|
||||
There are NO official/authorized resellers, because HTTrack is <b>NOT</b> a commercial product.
|
||||
But you can be charged for duplication fees, or any other services (example: software CDroms or shareware collections, or fees for maintenance),
|
||||
but you should have been informed that the software was free software/GPL, and you <b><u>MUST</u></b> have received a copy of the GNU General Public License.
|
||||
Otherwise this is dishonest and unfair (ie. selling httrack on ebay without telling that it was a free software is a scam).
|
||||
Otherwise this is dishonnest and unfair (ie. selling httrack on ebay without telling that it was a free software is a scam).
|
||||
</em>
|
||||
|
||||
<br><br><a NAME="QG0b">Q: <strong>Are there any risks of viruses with this software?</strong></a><br>
|
||||
A: <em>For the software itself:
|
||||
All official releases (at httrack.com) are checked against all known viruses, and the packaging process is also checked. Archives are stored on Un*x servers, not really concerned by viruses. It has been reported, however, that some rogue freeware sites are embedding free software and freeware inside badware installers. Always download httrack from the main site (www.httrack.com), and never from an untrusted source!<br>
|
||||
All official releases (at httrack.com) are checked against all known viruses, and the packaging process is also checked. Archives are stored on Un*x servers, not really concerned by viruses. It has been reported, however, that some rogue freeware sites are embedding free softwares and freewares inside badware installers. Always download httrack from the main site (www.httrack.com), and never from an untrusted source!<br>
|
||||
For files you are downloading on the WWW using HTTrack: You may encounter websites which were corrupted by viruses, and downloading data on these websites might be dangerous if you execute downloaded executables, or if embedded pages contain infected material (as dangerous as if using a regular Browser). Always ensure that websites you are crawling are safe.
|
||||
(Note: remember that using an antivirus software is a good idea once you are connected to the Internet)</em>
|
||||
|
||||
@@ -374,8 +376,11 @@ A: <em>That's right. You can, however, install WinHTTrack on your own machine, a
|
||||
<br><br><a NAME="QG2">Q: <strong>Where can I find French/other languages documentation?</strong></a><br>
|
||||
A: <em>Windows interface is available on several languages, but not yet the documentation!</em>
|
||||
|
||||
<br><br><a NAME="QG3">Q: <strong>Which systems does HTTrack run on?</strong></a><br>
|
||||
A: <em>HTTrack runs on current Windows, Linux and other Unix-like systems, and macOS. Very old platforms such as Windows 95/98 are no longer supported; you may try an older release (such as 3.33) on those.</em>
|
||||
<br><br><a NAME="QG3b">Q: <strong>Is HTTrack working on Windows Vista/Windows Seven/Windows 8 ?</strong></a><br>
|
||||
A: <em>Yes, it does</em>
|
||||
|
||||
<br><br><a NAME="QG3">Q: <strong>Is HTTrack working on Windows 95/98 ?</strong></a><br>
|
||||
A: <em>No, not anymore. You may try to pick an older release (such as 3.33)</em>
|
||||
|
||||
<br><br><a NAME="QG4">Q: <strong>What's the difference between HTTrack, WinHTTrack and WebHTTrack?</strong></a><br>
|
||||
A: <em>WinHTTrack is the Windows GUI release of HTTrack (with a native graphic shell) and WebHTTrack is the Linux/Posix release of HTTrack (with an html graphic shell)</em>
|
||||
@@ -389,7 +394,7 @@ A: <em>It should. The <tt>configure.ac</tt> may be modified in some cases, howev
|
||||
<br><br><a NAME="QG7">Q: <strong>I use HTTrack for professional purpose. What about restrictions/license fee?</strong></a><br>
|
||||
A: <em>HTTrack is covered by the GNU General Public License (GPL). There is no restrictions using HTTrack for professional purpose,
|
||||
except if you develop a software which uses HTTrack components (parts of the source, or any other component).
|
||||
See the <tt>license.txt</tt> file for more information</em>. See also the next question regarding copyright issues when redistributing downloaded material.
|
||||
See the <tt>license.txt</tt> file for more information</em>. See also the next question regarding copyright issues when reditributing downloaded material.
|
||||
|
||||
<br><br><a NAME="QG7b">Q: <strong>Is there any license royalties for distributing a mirror made with HTTrack?</strong></a><br>
|
||||
A: <em>On the HTTrack side, no. However, sharing, publishing or reusing copyrighted material downloaded from a site requires the authorization of the copyright holders, and possibly paying royalty fees. Always ask the authorization before creating a mirror of a site, even if the site appears to be royalty-free and/or without copyright notice.</em>
|
||||
@@ -410,7 +415,7 @@ There are several reasons (and solutions) for a mirror to fail. Reading the log
|
||||
|
||||
<ul>
|
||||
<li>Links within the site refers to external links, or links located in another (or upper) directories, not captured by default - the use of filters is generally THE solution, as this is one of the powerful option in HTTrack. <u>See the above questions/answers</u>.</li>
|
||||
<li>Website <a href="#Q1b1">'robots.txt' rules</a> forbid access to several website parts - you can disable them, but only with great care!</li>
|
||||
<li>Website <a href="#Q1b1">'robots.txt' rules</a> forbide access to several website parts - you can disable them, but only with great care!</li>
|
||||
<li>HTTrack is filtered (by its default User-agent IDentity) - you can change the Browser User-Agent identity to an anonymous one (MSIE, Netscape..) - here again, use this option with care, as this measure might have been put to avoid some bandwidth abuse (see also the <a href="abuse.html">abuse faq</a>!)</li>
|
||||
</ul>
|
||||
|
||||
@@ -452,14 +457,14 @@ A: <em>Yes, HTTrack does support (since 3.20 release) ipv6 sites, using A/AAAA e
|
||||
A: <em>Check the build options (you may have selected user-defined structure with wrong parameters!)</em>
|
||||
|
||||
<br><br><a NAME="QT5">Q: <strong>When capturing real audio/video links (.ram), I only get a shortcut!</a></strong></a></br>
|
||||
A: <em>The .ra/.rm associated file can be captured together with the shortcut, if proper filters are set. Streaming protocols such as rtsp:// are out of scope: HTTrack is an HTTP/FTP mirror and does not capture rtsp streams.</em>
|
||||
A: <em>Yes, but .ra/.rm associated file should be captured together - except if rtsp:// protocol is used (not supported by HTTrack yet), or if proper filters are needed</em>
|
||||
|
||||
<br><br><a NAME="QT6">Q: <strong>Using user:password@address is not working!</a></strong></a></br>
|
||||
A: <em>Again, first check the <tt>hts-log.txt</tt> and <tt>hts-err.txt</tt> error log files - this can give you precious information<br>
|
||||
The site may have a different authentication scheme - form based authentication, for example.
|
||||
In this case, use the URL capture features of HTTrack, it might work.
|
||||
<br>Note: If your username and/or password contains a '<tt>@</tt>' character, you may have to replace all '<tt>@</tt>'
|
||||
occurrences by '<tt>%40</tt>' so that it can work, such as in <tt>user%40domain.com:foobar@www.foo.com/auth/.
|
||||
occurences by '<tt>%40</tt>' so that it can work, such as in <tt>user%40domain.com:foobar@www.foo.com/auth/.
|
||||
You may have to do the same for all "special" characters like spaces (%20), quotes (%22)..</tt>
|
||||
</em>
|
||||
<br><br>
|
||||
@@ -515,7 +520,7 @@ These rules, stored in a file called robots.txt, are given by the website, to sp
|
||||
- for example, /cgi-bin or large images files.
|
||||
They are followed by default by HTTrack, as it is advised. Therefore, you may miss some files that would have been downloaded without
|
||||
these rules - check in your logs if it is the case:<br>
|
||||
<tt>Info: Note: due to www.foobar.com remote robots.txt rules, links beginning with these path will be forbidden: /cgi-bin/,/images/ (see in the options to disable this)
|
||||
<tt>Info: Note: due to www.foobar.com remote robots.txt rules, links begining with these path will be forbidden: /cgi-bin/,/images/ (see in the options to disable this)
|
||||
</tt>
|
||||
<br>
|
||||
If you want to disable them, just change the corresponding option in the option list! (but only disable this option with great care,
|
||||
@@ -535,7 +540,7 @@ HTTrack must find one. Therefore, two index.html will be produced, one with the
|
||||
<br>
|
||||
It might be a good idea to consider that http://www.foobar.com/ and http://www.foobar.com/index.html are the same links, to avoid
|
||||
duplicate files, isn't it?
|
||||
NO, because the top index (/) can refer to ANY filename, and if index.html is generally the default name, index.htm can be chosen,
|
||||
NO, because the top index (/) can refer to ANY filename, and if index.html is generally the default name, index.htm can be choosen,
|
||||
or index.php3, mydog.jpg, or anything you may imagine. (some webmasters are really crazy)
|
||||
<br>
|
||||
<br>
|
||||
@@ -590,7 +595,8 @@ A: <em>Simply use the <tt>--assume dat=application/x-zip</tt> option
|
||||
A: <em>You may need cookies! Cookies are specific data (for example, your username or password) that are sent to your browser once
|
||||
you have logged in certain sites so that you only have to log-in once. For example, after having entered your username in a website, you can
|
||||
view pages and articles, and the next time you will go to this site, you will not have to re-enter your username/password.<br>
|
||||
To supply your own cookies to an HTTrack project, put a Netscape-format <tt>cookies.txt</tt> file in your project folder, or point HTTrack at one with the <tt>--cookies-file</tt> option. You can export your browser's session cookies to that format with a browser extension.
|
||||
To "merge" your personnal cookies to an HTTrack project, just copy the cookies.txt file from your Netscape folder (or the cookies located into the Temporary Internet Files folder for IE)
|
||||
into your project folder (or even the HTTrack folder)
|
||||
</em>
|
||||
<br>
|
||||
<br>
|
||||
@@ -655,7 +661,7 @@ Shell version, skip some slow files, too.</em><br>
|
||||
|
||||
</a><a NAME="Q3b">Q: <strong>I want to update a site, but it's taking too much time! What's happening?</strong><br>
|
||||
A: <em>First, HTTrack always tries to minimize the download flow by interrogating the server about the
|
||||
file changes. But, because HTTrack has to rescan all files from the beginning to rebuild the local site structure,
|
||||
file changes. But, because HTTrack has to rescan all files from the begining to rebuild the local site structure,
|
||||
it can take some time.
|
||||
Besides, some servers are not very smart and always consider that they get newer files, forcing HTTrack to reload them,
|
||||
even if no changes have been made!
|
||||
@@ -734,7 +740,7 @@ retransferred.</em><br>
|
||||
</a><a NAME="Q7">Q: <strong>I just want to retrieve all ZIP files or other files in a web
|
||||
site/in a page. How do I do it?</strong><br>
|
||||
A: <em>You can use different methods. You can use the 'get files near a link' option if
|
||||
files are in a foreign domain. You can use, too, a filter address: adding <tt>+*.zip</tt>
|
||||
files are in a foreign domain. You can use, too, a filter adress: adding <tt>+*.zip</tt>
|
||||
in the URL list (or in the filter list) will accept all ZIP files, even if these files are
|
||||
outside the address. <br>
|
||||
Example : <tt>httrack www.example.com/someaddress.html +*.zip</tt> will allow
|
||||
|
||||
@@ -111,7 +111,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
starts links, the default mode is to mirror these links - i.e. if one of your start page is
|
||||
www.example.com/test/index.html, all links starting with www.example.com/test/ will be
|
||||
accepted. But links directly in www.example.com/.. will not be accepted, however, because
|
||||
they are in a higher structure. This prevent HTTrack from mirroring the whole site. (All
|
||||
they are in a higher strcuture. This prevent HTTrack from mirroring the whole site. (All
|
||||
files in structure levels equal or lower than the primary links will be retrieved.)<br>
|
||||
</i>
|
||||
<br>
|
||||
@@ -127,15 +127,13 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
an authorization filter, like <b><tt>+*.gif</tt></b>. The pattern is a plus (this one: <b><tt>+</tt></b>),
|
||||
followed by a pattern composed of letters and wildcards (this one: <b><tt>*</tt></b>).
|
||||
<br><br>
|
||||
To forbid a family of links, define
|
||||
To forbide a family of links, define
|
||||
an authorization filter, like <b><tt>-*.gif</tt></b>. The pattern is a dash (this one: <b><tt>-</tt></b>),
|
||||
followed by a the same kind of pattern as for the authorization filter.
|
||||
<br><br>
|
||||
Example: +*.gif will accept all files finished by .gif<br>
|
||||
Example: -*.gif will refuse all files finished by .gif<br>
|
||||
<br>
|
||||
To see which rule accepted or blocked a given URL, run HTTrack with the <b><tt>--why</tt></b> (<b><tt>-%Y</tt></b>) option, described in <a href="httrack.man.html#OPTIONS">the manual page</a>.<br>
|
||||
<br>
|
||||
|
||||
<p>
|
||||
<h4>Scan rules based on size (e.g. accept or refuse files bigger/smaller than a certain size)</h4>
|
||||
@@ -145,7 +143,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
size to ensure that you won't reach a defined limit.
|
||||
|
||||
Example: You may want to accept all files on the domain www.example.com, using '+www.example.com/*',
|
||||
including gif files inside this domain and outside (external images), but not take to large images,
|
||||
including gif files inside this domain and outside (eternal images), but not take to large images,
|
||||
or too small ones (thumbnails)<br>
|
||||
Excluding gif images smaller than 5KB and images larger than 100KB is therefore a good option;
|
||||
+www.example.com +*.gif -*.gif*[<5] -*.gif*[>100]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -109,12 +109,12 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li>How to Use</li>
|
||||
<ul>
|
||||
<li><a href="shelldoc.html">WinHTTrack/WebHTTrack (GUI version for Windows or Linux/Unix)</a></li>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix release (WebHTTrack)<br>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix relese (WebHTTrack)<br>
|
||||
<br>
|
||||
<li><a href="android.html">HTTrack on Android</a></li>
|
||||
<br>Step-by-step guide for the Android app<br>
|
||||
<br>
|
||||
<li><a href="httrack.man.html">Command-line version</a></li>
|
||||
<li><a href="cmddoc.html">Command-line version</a></li>
|
||||
<br>How to run HTTrack from the command line, and where to find every option<br>
|
||||
<br>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
@@ -123,7 +123,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li><a href="fcguide.html">HTTrack Users Guide By Fred Cohen</a></li>
|
||||
<br>A tutorial that describes all command-line options, for Linux and Windows users<br>
|
||||
<br>
|
||||
<li><a href="dev.html">Developer/Programming</a></li>
|
||||
<li><a href="dev.html">Developper/Programming</a></li>
|
||||
<br>How to use HTTrack in batch files, and how to use the library<br>
|
||||
</ul>
|
||||
<br>
|
||||
|
||||
@@ -101,10 +101,6 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<h2 align="center"><em>Options</em></h2>
|
||||
|
||||
<p><strong>Note:</strong> this is a legacy option list and may be out of date.
|
||||
For the authoritative, always-current reference see <a href="httrack.man.html">the httrack manual page</a>,
|
||||
and for a task-oriented walkthrough see the <a href="cmdguide.html">Command-line Guide</a>.</p>
|
||||
|
||||
<ul>
|
||||
<li>Filters: <a href="filters.html">how to use them</a></li>
|
||||
<br><small>Here you can find informations on filters: how to accept all gif files in a mirror, for example</small>
|
||||
|
||||
@@ -95,7 +95,7 @@ ${do:end-if}
|
||||
|
||||
<table border="0" width="100%">
|
||||
<tr><td width="90%">
|
||||
<h2 align="center"><em>${LANG_G44}</em></h2>
|
||||
<h2 align="center"><em>Select URLs</em></h2>
|
||||
</td>
|
||||
${/* show help only if available */}
|
||||
${do:if-file-exists:html/index.html}
|
||||
|
||||
@@ -77,7 +77,7 @@ ${do:end-if}
|
||||
|
||||
<table border="0" width="100%">
|
||||
<tr><td width="90%">
|
||||
<h2 align="center"><em>${LANG_J9}</em></h2>
|
||||
<h2 align="center"><em>Start</em></h2>
|
||||
</td>
|
||||
${/* show help only if available */}
|
||||
${do:if-file-exists:html/index.html}
|
||||
|
||||
@@ -16,11 +16,6 @@ regen-man: makeman.sh $(top_builddir)/src/httrack$(EXEEXT)
|
||||
|
||||
# Render html/httrack.man.html from httrack.1. Needs the groff html device
|
||||
# (Debian: full "groff" package, not "groff-base"). Run by hand: make -C man regen-man-html
|
||||
# Strip groff's version-stamp and creation-date comments so the committed file
|
||||
# doesn't churn across groff versions or rebuilds; the ci man-page-sync guard
|
||||
# only requires this file to change whenever httrack.1 does.
|
||||
regen-man-html: httrack.1
|
||||
groff -t -man -Thtml $(srcdir)/httrack.1 \
|
||||
| sed -e 's/groff version [0-9][0-9.]*/groff/' -e '/^<!-- CreationDate:/d' \
|
||||
> $(top_srcdir)/html/httrack.man.html
|
||||
groff -t -man -Thtml $(srcdir)/httrack.1 > $(top_srcdir)/html/httrack.man.html
|
||||
.PHONY: regen-man-html
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "22 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "21 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -37,7 +37,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%L, \-\-list\fR ]
|
||||
[ \fB\-%S, \-\-urllist\fR ]
|
||||
[ \fB\-NN, \-\-structure[=N]\fR ]
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
@@ -58,7 +57,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-sN, \-\-robots[=N]\fR ]
|
||||
[ \fB\-%h, \-\-http\-10\fR ]
|
||||
[ \fB\-%k, \-\-keep\-alive\fR ]
|
||||
[ \fB\-%z, \-\-disable\-compression\fR ]
|
||||
[ \fB\-%B, \-\-tolerant\fR ]
|
||||
[ \fB\-%s, \-\-updatehack\fR ]
|
||||
[ \fB\-%u, \-\-urlhack\fR ]
|
||||
@@ -99,7 +97,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%!, \-\-disable\-security\-limits\fR ]
|
||||
[ \fB\-V, \-\-userdef\-cmd\fR ]
|
||||
[ \fB\-%W, \-\-callback\fR ]
|
||||
[ \fB\-y, \-\-background\-on\-suspend\fR ]
|
||||
[ \fB\-K, \-\-keep\-links[=N]\fR ]
|
||||
.SH DESCRIPTION
|
||||
.B httrack
|
||||
@@ -192,13 +189,11 @@ structure type (0 *original structure, 1+: see below) (\-\-structure[=N])
|
||||
.br
|
||||
or user defined structure (\-N "%h%p/%n%q.%t")
|
||||
.IP \-%N
|
||||
delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use) (\-\-delayed\-type\-check)
|
||||
delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use)
|
||||
.IP \-%D
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
long names (L1 *long names / L0 8\-3 conversion / L2 ISO9660 compatible) (\-\-long\-names[=N])
|
||||
.IP \-KN
|
||||
@@ -236,8 +231,6 @@ follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always
|
||||
force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (\-\-http\-10)
|
||||
.IP \-%k
|
||||
use keep\-alive if possible, greately reducing latency for small files and test requests (%k0 don't use) (\-\-keep\-alive)
|
||||
.IP \-%z
|
||||
do not request compressed content (%z0 request) (\-\-disable\-compression)
|
||||
.IP \-%B
|
||||
tolerant requests (accept bogus responses on some servers, but not standard!) (\-\-tolerant)
|
||||
.IP \-%s
|
||||
@@ -332,6 +325,8 @@ go everywhere on the web (\-\-go\-everywhere)
|
||||
.IP \-%H
|
||||
debug HTTP headers in logfile (\-\-debug\-headers)
|
||||
.SS Guru options: (do NOT use if possible)
|
||||
.IP \-#X
|
||||
*use optimized engine (limited memory boundary checks) (\-\-fast\-engine)
|
||||
.IP \-#test
|
||||
list engine self\-tests (run one with \-#test=NAME [args])
|
||||
.IP \-#C
|
||||
@@ -356,6 +351,8 @@ maximum number of links (\-#L1000000) (\-\-advanced\-maxlinks[=N])
|
||||
display ugly progress information (\-\-advanced\-progressinfo)
|
||||
.IP \-#P
|
||||
catch URL (\-\-catch\-url)
|
||||
.IP \-#R
|
||||
old FTP routines (debug) (\-\-repair\-cache)
|
||||
.IP \-#T
|
||||
generate transfer ops. log every minutes (\-\-debug\-xfrstats)
|
||||
.IP \-#u
|
||||
@@ -374,8 +371,6 @@ USE IT WITH EXTREME CARE
|
||||
execute system command after each files ($0 is the filename: \-V "rm \\$0") (\-\-userdef\-cmd <param>)
|
||||
.IP \-%W
|
||||
use an external library function as a wrapper (\-%W myfoo.so[,myparameters]) (\-\-callback <param>)
|
||||
.IP \-y
|
||||
go to background when suspended (y0 don't) (\-\-background\-on\-suspend)
|
||||
.SS Details: Option N
|
||||
.IP \-N0
|
||||
Site\-structure (default)
|
||||
|
||||
220
src/htscache.c
220
src/htscache.c
@@ -856,41 +856,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
// si save==null alors test unqiquement
|
||||
static int hts_rename(httrackp * opt, const char *a, const char *b) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: rename %s -> %s (%p %p)", a, b, a, b);
|
||||
return RENAME(a, b);
|
||||
}
|
||||
|
||||
/* Open the cache ZIP via hts_fopen_utf8 so a non-ASCII path_log isn't mangled
|
||||
to ANSI (#630); 64-bit funcs keep multi-GB caches whole on Windows LLP64. */
|
||||
static voidpf ZCALLBACK hts_zip_fopen_utf8(voidpf opaque, const void *filename,
|
||||
int mode) {
|
||||
const char *mode_fopen = NULL;
|
||||
|
||||
(void) opaque;
|
||||
if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER) == ZLIB_FILEFUNC_MODE_READ)
|
||||
mode_fopen = "rb";
|
||||
else if (mode & ZLIB_FILEFUNC_MODE_EXISTING)
|
||||
mode_fopen = "r+b";
|
||||
else if (mode & ZLIB_FILEFUNC_MODE_CREATE)
|
||||
mode_fopen = "wb";
|
||||
if (filename == NULL || mode_fopen == NULL)
|
||||
return NULL;
|
||||
return (voidpf) FOPEN((const char *) filename, mode_fopen);
|
||||
}
|
||||
|
||||
static unzFile hts_unzOpen_utf8(const char *path) {
|
||||
zlib_filefunc64_def ff;
|
||||
|
||||
fill_fopen64_filefunc(&ff);
|
||||
ff.zopen64_file = hts_zip_fopen_utf8;
|
||||
return unzOpen2_64(path, &ff);
|
||||
}
|
||||
|
||||
static zipFile hts_zipOpen_utf8(const char *path, int append) {
|
||||
zlib_filefunc64_def ff;
|
||||
|
||||
fill_fopen64_filefunc(&ff);
|
||||
ff.zopen64_file = hts_zip_fopen_utf8;
|
||||
return zipOpen2_64(path, append, NULL, &ff);
|
||||
return rename(a, b);
|
||||
}
|
||||
|
||||
/* Pathname of a file inside the mirror dir (rotating concat buffer). */
|
||||
@@ -908,9 +874,9 @@ static char *reconcile_path(httrackp *opt, const char *name) {
|
||||
/* Replace the new-generation file by the old one, when the old one exists. */
|
||||
static void reconcile_promote(httrackp *opt, const char *oldname,
|
||||
const char *newname) {
|
||||
if (fexist_utf8(reconcile_path(opt, oldname))) {
|
||||
UNLINK(reconcile_path(opt, newname));
|
||||
RENAME(reconcile_path(opt, oldname), reconcile_path(opt, newname));
|
||||
if (fexist(reconcile_path(opt, oldname))) {
|
||||
remove(reconcile_path(opt, newname));
|
||||
rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -919,7 +885,7 @@ void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
|
||||
case CACHE_RECONCILE_PROMOTE:
|
||||
/* Previous run rotated new.* to old.* then died before writing: promote
|
||||
the old generation back, whichever format it uses. */
|
||||
if (!fexist_utf8(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
break;
|
||||
case CACHE_RECONCILE_INTERRUPTED:
|
||||
@@ -928,17 +894,16 @@ void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
|
||||
is -1 for a missing file, which would spuriously pass the "< TINY" test
|
||||
and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
|
||||
*/
|
||||
if (!opt->cache ||
|
||||
!fexist_utf8(reconcile_path(opt, "hts-in_progress.lock")))
|
||||
if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
|
||||
break;
|
||||
if (fexist_utf8(reconcile_path(opt, "hts-cache/new.zip")) &&
|
||||
fexist_utf8(reconcile_path(opt, "hts-cache/old.zip")) &&
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/new.zip")) <
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
break;
|
||||
case CACHE_RECONCILE_ROLLBACK:
|
||||
@@ -975,26 +940,24 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
#endif
|
||||
if (!cache->ro) {
|
||||
#ifdef _WIN32
|
||||
/* Windows mkdir takes no mode; use the UTF-8 wrapper for #630. */
|
||||
MKDIR(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache"));
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"));
|
||||
#else
|
||||
/* keep the cache dir 0700, not MKDIR's 0755. */
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache"),
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
|
||||
HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
if ((fexist_utf8(fconcat(
|
||||
if ((fexist(fconcat(
|
||||
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))) { // a previous cache exists.. rename it
|
||||
/* Remove OLD cache */
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")) !=
|
||||
0) {
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"))) {
|
||||
if (remove
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) != 0) {
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
|
||||
"Cache: error while moving previous cache");
|
||||
}
|
||||
@@ -1017,27 +980,33 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: no cache found");
|
||||
}
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: size %d",
|
||||
(int) fsize_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")));
|
||||
(int)
|
||||
fsize(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")));
|
||||
|
||||
// charger index cache précédent
|
||||
if ((!cache->ro &&
|
||||
fsize_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip")) >
|
||||
0) ||
|
||||
(cache->ro &&
|
||||
fsize_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip")) >
|
||||
0)) {
|
||||
if ((!cache->ro
|
||||
&&
|
||||
fsize(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip")) > 0)
|
||||
|| (cache->ro
|
||||
&&
|
||||
fsize(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")) > 0)
|
||||
) {
|
||||
if (!cache->ro) {
|
||||
cache->zipInput = hts_unzOpen_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.zip"));
|
||||
cache->zipInput =
|
||||
unzOpen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.zip"));
|
||||
} else {
|
||||
cache->zipInput = hts_unzOpen_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"));
|
||||
cache->zipInput =
|
||||
unzOpen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"));
|
||||
}
|
||||
|
||||
// Corrupted ZIP file ? Try to repair!
|
||||
@@ -1057,8 +1026,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
}
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Cache: damaged cache, trying to repair");
|
||||
/* mztools has no UTF-8 hook, so repairing a corrupt cache under a
|
||||
non-ASCII path_log fails cleanly (re-crawl), never forks a twin. */
|
||||
if (unzRepair
|
||||
(name,
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
@@ -1066,11 +1033,11 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/repair.tmp"),
|
||||
&repaired, &repairedBytes) == Z_OK) {
|
||||
UNLINK(name);
|
||||
RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/repair.zip"),
|
||||
name);
|
||||
cache->zipInput = hts_unzOpen_utf8(name);
|
||||
unlink(name);
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/repair.zip"), name);
|
||||
cache->zipInput = unzOpen(name);
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Cache: %d bytes successfully recovered in %d entries",
|
||||
(int) repairedBytes, (int) repaired);
|
||||
@@ -1166,8 +1133,8 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"Cache: error trying to open the cache");
|
||||
}
|
||||
|
||||
} else if (fsize_utf8(reconcile_path(opt, "hts-cache/old.ndx")) > 0 ||
|
||||
fsize_utf8(reconcile_path(opt, "hts-cache/new.ndx")) > 0) {
|
||||
} else if (fsize(reconcile_path(opt, "hts-cache/old.ndx")) > 0 ||
|
||||
fsize(reconcile_path(opt, "hts-cache/new.ndx")) > 0) {
|
||||
/* pre-3.31 (2003) .dat/.ndx cache: import support removed */
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Cache: the pre-3.31 .dat/.ndx cache format is no longer "
|
||||
@@ -1183,58 +1150,67 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
#endif
|
||||
if (!cache->ro) {
|
||||
// ouvrir caches actuels
|
||||
structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/"));
|
||||
structcheck(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/"));
|
||||
|
||||
{
|
||||
/* Create ZIP file cache */
|
||||
cache->zipOutput = (void *) hts_zipOpen_utf8(
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.zip"),
|
||||
0);
|
||||
cache->zipOutput =
|
||||
(void *)
|
||||
zipOpen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip"), 0);
|
||||
|
||||
if (cache->zipOutput != NULL) {
|
||||
// supprimer old.lst
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.lst"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"));
|
||||
// renommer
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst")))
|
||||
RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst"),
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.lst"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst")))
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"));
|
||||
// ouvrir
|
||||
cache->lst =
|
||||
FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst"),
|
||||
"wb");
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst"), "wb");
|
||||
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
|
||||
opt->state.strc.lst = cache->lst;
|
||||
|
||||
// supprimer old.txt
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt")))
|
||||
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt"));
|
||||
// renommer
|
||||
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt")))
|
||||
RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.txt"),
|
||||
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/old.txt"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt")))
|
||||
rename(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.txt"));
|
||||
// ouvrir
|
||||
cache->txt =
|
||||
FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.txt"),
|
||||
"wb");
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.txt"), "wb");
|
||||
if (cache->txt) {
|
||||
fprintf(cache->txt,
|
||||
"date\tsize'/'remotesize\tflags(request:Update,Range state:File response:Modified,Chunked,gZipped)\t");
|
||||
|
||||
@@ -77,17 +77,17 @@ void infomsg(const char *msg) {
|
||||
if (msg[2] != ' ') {
|
||||
if ((msg[3] == ' ') || (msg[4] == ' ')) {
|
||||
char cmd[32] = "-";
|
||||
int p;
|
||||
int p = 0;
|
||||
|
||||
sscanf(msg, "%30s", cmd + strlen(cmd));
|
||||
/* try the flag as-is, then strip a trailing N as the numeric-arg
|
||||
placeholder (cN -> c); this order keeps -%N from becoming -% */
|
||||
while(cmd[p] == ' ')
|
||||
p++;
|
||||
sscanf(msg + p, "%s", cmd + strlen(cmd));
|
||||
/* clears cN -> c */
|
||||
if ((p = (int) strlen(cmd)) > 2)
|
||||
if (cmd[p - 1] == 'N')
|
||||
cmd[p - 1] = '\0';
|
||||
/* finds alias (if any) */
|
||||
p = optreal_find(cmd);
|
||||
if (p < 0 && (int) strlen(cmd) > 2 &&
|
||||
cmd[strlen(cmd) - 1] == 'N') {
|
||||
cmd[strlen(cmd) - 1] = '\0';
|
||||
p = optreal_find(cmd);
|
||||
}
|
||||
if (p >= 0) {
|
||||
/* fings type of parameter: number,param,param concatenated,single cmd */
|
||||
if (strcmp(opttype_value(p), "param") == 0)
|
||||
@@ -524,8 +524,6 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
(" LN long names (L1 *long names / L0 8-3 conversion / L2 ISO9660 compatible)");
|
||||
infomsg
|
||||
@@ -556,7 +554,6 @@ void help(const char *app, int more) {
|
||||
(" %h force HTTP/1.0 requests (reduce update features, only for old servers or proxies)");
|
||||
infomsg
|
||||
(" %k use keep-alive if possible, greately reducing latency for small files and test requests (%k0 don't use)");
|
||||
infomsg(" %z do not request compressed content (%z0 request)");
|
||||
infomsg
|
||||
(" %B tolerant requests (accept bogus responses on some servers, but not standard!)");
|
||||
infomsg
|
||||
@@ -623,6 +620,7 @@ void help(const char *app, int more) {
|
||||
infomsg(" %H debug HTTP headers in logfile");
|
||||
infomsg("");
|
||||
infomsg("Guru options: (do NOT use if possible)");
|
||||
infomsg(" #X *use optimized engine (limited memory boundary checks)");
|
||||
infomsg(" #test list engine self-tests (run one with -#test=NAME [args])");
|
||||
infomsg(" #C cache list (-#C '*.com/spider*.gif'");
|
||||
infomsg(" #R cache repair (damaged cache)");
|
||||
@@ -635,6 +633,7 @@ void help(const char *app, int more) {
|
||||
infomsg(" #L maximum number of links (-#L1000000)");
|
||||
infomsg(" #p display ugly progress information");
|
||||
infomsg(" #P catch URL");
|
||||
infomsg(" #R old FTP routines (debug)");
|
||||
infomsg(" #T generate transfer ops. log every minutes");
|
||||
infomsg(" #u wait time");
|
||||
infomsg(" #Z generate transfer rate statistics every minutes");
|
||||
@@ -649,9 +648,9 @@ void help(const char *app, int more) {
|
||||
infomsg("Command-line specific options:");
|
||||
infomsg
|
||||
(" V execute system command after each files ($0 is the filename: -V \"rm \\$0\")");
|
||||
infomsg(" %W use an external library function as a wrapper (-%W "
|
||||
"myfoo.so[,myparameters])");
|
||||
infomsg(" y go to background when suspended (y0 don't)");
|
||||
infomsg
|
||||
(" %W use an external library function as a wrapper (-%W myfoo.so[,myparameters])");
|
||||
/* infomsg(" %O do a chroot before setuid"); */
|
||||
infomsg("");
|
||||
infomsg("Details: Option N");
|
||||
infomsg(" N0 Site-structure (default)");
|
||||
|
||||
@@ -315,18 +315,9 @@ static void escape_url_parens(char *const s, const size_t size) {
|
||||
strlcpybuff(s, buff, size);
|
||||
}
|
||||
|
||||
/* Default port for lien's scheme (case-insensitive), 80 if absent/unknown; so
|
||||
schemeless and protocol-relative //host links default to 80 (known gap). */
|
||||
static int scheme_default_port(const char *lien) {
|
||||
if (strfield(lien, "https:"))
|
||||
return 443;
|
||||
if (strfield(lien, "ftp:"))
|
||||
return 21;
|
||||
return 80;
|
||||
}
|
||||
|
||||
/* Strip the scheme's own default port (80 http, 443 https, 21 ftp) from lien's
|
||||
authority in place; :80 on https/ftp stays as a real port (#638, #614). */
|
||||
/* Strip a default ":80" from lien's authority in place. Any spelling that
|
||||
range-parses to 80 (":80", ":080") is dropped by its matched length, not a
|
||||
hardcoded 3 chars; a value that merely wraps to 80 as an int (#614) stays. */
|
||||
void hts_strip_default_port(char *lien, size_t size) {
|
||||
char *a;
|
||||
|
||||
@@ -348,8 +339,7 @@ void hts_strip_default_port(char *lien, size_t size) {
|
||||
b++;
|
||||
saved = *b;
|
||||
*b = '\0';
|
||||
is_default =
|
||||
hts_parse_url_port(a + 1, &port) && port == scheme_default_port(lien);
|
||||
is_default = hts_parse_url_port(a + 1, &port) && port == 80;
|
||||
*b = saved;
|
||||
if (is_default) { // default port, strip it
|
||||
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
|
||||
|
||||
@@ -1589,8 +1589,10 @@ static int st_identabs(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Default-port strip is scheme-aware (#638), overflow-safe (#614): a scheme's
|
||||
own default (any spelling) is dropped, a real port stays; guards #627. */
|
||||
/* Default-port strip (#627): a genuine 80 (any spelling) is removed by its
|
||||
matched length, host preserved; a non-80 port or one that only wraps to 80 as
|
||||
a 32-bit int (#614) is left intact. Guards the old bug where ":080"/":0080"
|
||||
dropped a hardcoded 3 chars and glued the leftover digits onto the host. */
|
||||
static int st_stripport(httrackp *opt, int argc, char **argv) {
|
||||
static const struct {
|
||||
const char *in, *out;
|
||||
@@ -1604,13 +1606,6 @@ static int st_stripport(httrackp *opt, int argc, char **argv) {
|
||||
{"http://127.0.0.1:8080/x", "http://127.0.0.1:8080/x"},
|
||||
{"http://127.0.0.1:4294967376/x", "http://127.0.0.1:4294967376/x"},
|
||||
{"http://127.0.0.1/x", "http://127.0.0.1/x"},
|
||||
{"https://127.0.0.1:443/x", "https://127.0.0.1/x"},
|
||||
{"https://127.0.0.1:80/x", "https://127.0.0.1:80/x"},
|
||||
// Scheme match is case-insensitive: HTTPS' default is 443, so :80 stays.
|
||||
{"HTTPS://127.0.0.1:80/x", "HTTPS://127.0.0.1:80/x"},
|
||||
{"ftp://127.0.0.1:21/x", "ftp://127.0.0.1/x"},
|
||||
{"ftp://127.0.0.1:80/x", "ftp://127.0.0.1:80/x"},
|
||||
{"http://127.0.0.1:443/x", "http://127.0.0.1:443/x"},
|
||||
};
|
||||
|
||||
size_t k;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
--- mztools.c.orig 2024-01-27 14:07:18.636193212 +0100
|
||||
+++ mztools.c 2024-01-27 14:09:55.356620093 +0100
|
||||
@@ -10,10 +10,11 @@
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <string.h>
|
||||
#include "zlib.h"
|
||||
#include "unzip.h"
|
||||
@@ -8,11 +8,6 @@
|
||||
|
||||
#define READ_8(adr) ((unsigned char)*(adr))
|
||||
#define READ_16(adr) ( READ_8(adr) | (READ_8(adr+1) << 8) )
|
||||
-#define READ_32(adr) ( READ_16(adr) | (READ_16((adr)+2) << 16) )
|
||||
+#define READ_32(adr) ((uLong) READ_16(adr) | ((uLong) READ_16((adr) + 2) << 16))
|
||||
|
||||
#define WRITE_8(buff, n) do { \
|
||||
*((unsigned char*)(buff)) = (unsigned char) ((n) & 0xff); \
|
||||
@@ -141,8 +142,8 @@
|
||||
/* Central directory entry */
|
||||
{
|
||||
|
||||
@@ -6,7 +6,6 @@ set -euo pipefail
|
||||
# Stripping a default :80 from a crawled link used to skip a hardcoded 3 chars
|
||||
# (#627): ":080"/":0080" lost only ":80" and glued the rest onto the host
|
||||
# (127.0.0.1:080/x -> 127.0.0.10/x), and a value wrapping to 80 as a 32-bit int
|
||||
# (#614) was stripped as if it were the default. The default is scheme-aware
|
||||
# (#638): an explicit :80 on https/ftp stays, :443/:21 strip under their own
|
||||
# scheme. All assertions live in the engine self-test.
|
||||
# (#614) was stripped as if it were the default. All assertions live in the
|
||||
# engine self-test.
|
||||
httrack -O /dev/null -#test=stripport | grep -q "stripport self-test OK"
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The
|
||||
# logs (hts-log.txt/hts-err.txt) and the hts-cache must land there, not in an
|
||||
# ANSI-mangled twin: on Windows path_log holds UTF-8 bytes the raw file calls
|
||||
# read as the codepage. The --log-found audit greps logroot=café/hts-log.txt and
|
||||
# --cache-under-logroot checks café/hts-cache/new.zip, so anything written to the
|
||||
# twin fails them. POSIX has no twin, so this bites on the Windows CI leg (test
|
||||
# 64), exercising only the code path elsewhere.
|
||||
# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin:
|
||||
# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage.
|
||||
# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so
|
||||
# a log written to the twin fails them. POSIX has no twin, so this bites on the
|
||||
# Windows CI leg (like test 64). The cache twin is a separate, larger fix.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \
|
||||
--found 'simple/basic.html' \
|
||||
--log-found 'mirror complete in' \
|
||||
--cache-under-logroot \
|
||||
httrack 'BASEURL/simple/basic.html'
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Control for wait_bounded: a wedged fetch must be reaped at the harness deadline,
|
||||
# not hang the CI step to its 45-min cap.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
start=$SECONDS
|
||||
out=$(CRAWL_DEADLINE=3 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
httrack 'BASEURL/watchdog/stall' 2>&1) && rc=0 || rc=$?
|
||||
elapsed=$((SECONDS - start))
|
||||
|
||||
# Skip (not fail) on causes unrelated to the watchdog: no python3/server, or a
|
||||
# port-announce race (flaky-13).
|
||||
test "$rc" -ne 77 || {
|
||||
echo "SKIP: local crawl prerequisites missing" >&2
|
||||
exit 77
|
||||
}
|
||||
if printf '%s\n' "$out" | grep -q "could not discover server port"; then
|
||||
echo "SKIP: server port announce raced" >&2
|
||||
exit 77
|
||||
fi
|
||||
|
||||
# The message prints only on the 124 reap, and 25s < the engine's --timeout=30, so
|
||||
# together they pin the fast exit on the 3s watchdog, not httrack giving up.
|
||||
test "$rc" -ne 0 || fail "stalled crawl reported success"
|
||||
test "$elapsed" -lt 25 || fail "watchdog fired late (${elapsed}s)"
|
||||
printf '%s\n' "$out" | grep -q "watchdog fired" ||
|
||||
fail "crawl failed but not via the watchdog: $out"
|
||||
|
||||
echo "crawl watchdog OK (reaped in ${elapsed}s)"
|
||||
@@ -152,7 +152,6 @@ TESTS = \
|
||||
67_engine-delayed-truncate.test \
|
||||
68_webhttrack-outdir-charset.test \
|
||||
69_local-intl-logdir.test \
|
||||
71_local-crange-repaircache.test \
|
||||
72_watchdog-crawl.test
|
||||
71_local-crange-repaircache.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -118,9 +118,6 @@ while test "$pos" -lt "$nargs"; do
|
||||
nopurge=1
|
||||
audit+=("--no-purge")
|
||||
;;
|
||||
--cache-under-logroot)
|
||||
audit+=("--cache-under-logroot")
|
||||
;;
|
||||
--tls)
|
||||
tls=1
|
||||
scheme=https
|
||||
@@ -243,17 +240,13 @@ fi
|
||||
# Localhost is fast; disable the rate/bandwidth safety limits but keep a
|
||||
# max-time backstop so a hang cannot wedge the suite.
|
||||
declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
|
||||
# Watchdog above --max-time so the engine limit fires first when healthy; only a
|
||||
# genuine wedge trips it. CRAWL_DEADLINE lets 72_watchdog-crawl drive it low.
|
||||
crawl_deadline=${CRAWL_DEADLINE:-180}
|
||||
log="${tmpdir}/log"
|
||||
info "running httrack ${hts[*]}"
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline"
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
crawlpid=
|
||||
test "$crawlres" -ne 124 || warning "crawl watchdog fired after ${crawl_deadline}s"
|
||||
# httrack exits 0 even on hard connect/DNS errors, so this is a backstop only;
|
||||
# the real guard is the audit below (--errors 0 plus the host-root existence check).
|
||||
test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
|
||||
@@ -269,7 +262,7 @@ if test -n "$rerun"; then
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline"
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
crawlpid=
|
||||
test "$crawlres" -eq 0 || ! result "update pass exited $crawlres" || {
|
||||
@@ -297,7 +290,7 @@ if test -n "$rerun_args"; then
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline"
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
crawlpid=
|
||||
test "$crawlres" -eq 0 || ! result "second pass exited $crawlres" || {
|
||||
@@ -319,7 +312,7 @@ if test -n "$rerun_dead"; then
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline" || true
|
||||
wait "$crawlpid" || true
|
||||
crawlpid=
|
||||
result "OK (dead pass ran)"
|
||||
# The dead pass must have gone through the no-data rollback, not bailed out
|
||||
@@ -386,15 +379,6 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
|
||||
assert_equals "checking files" "${audit[$i]}" "$nFiles"
|
||||
;;
|
||||
--cache-under-logroot)
|
||||
# The cache must sit under path_log (logroot), not an ANSI-mangled twin
|
||||
# of a non-ASCII -O dir (#630 cache half). Bites only on the Windows leg.
|
||||
info "checking cache under logroot"
|
||||
if test -e "${logroot}/hts-cache/new.zip"; then result "OK"; else
|
||||
result "cache not under logroot (mangled twin?)"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
--found)
|
||||
i=$((i + 1))
|
||||
info "checking for ${audit[$i]}"
|
||||
|
||||
@@ -979,22 +979,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
# Always-stall endpoint for 72_watchdog-crawl: never finishes, so the harness
|
||||
# watchdog must reap it.
|
||||
def route_watchdog_stall(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", "1000000") # never delivered
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(b"STALL")
|
||||
self.wfile.flush()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# C7: stall the first GET (partial + temp-ref), then answer the resume's
|
||||
# Range with a bogus 304; httrack must drop the partial and refetch.
|
||||
RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
|
||||
@@ -1579,7 +1563,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/types/gen.php": route_types,
|
||||
"/intl/index.html": route_intl_index,
|
||||
"/intl/" + INTL_NAME: route_intl_page,
|
||||
"/watchdog/stall": route_watchdog_stall,
|
||||
"/resume/index.html": route_resume_index,
|
||||
"/resume/blob.txt": route_resume,
|
||||
"/resume304/index.html": route_resume304_index,
|
||||
|
||||
@@ -31,13 +31,15 @@ is_windows() {
|
||||
esac
|
||||
}
|
||||
|
||||
# On Windows MSYS can't signal a native python.exe, so kill_tree ends the whole
|
||||
# tree (a bare kill -9 leaves children). "|| true" throughout: callers run under
|
||||
# set -e and the reap makes wait return 143.
|
||||
# Stop a backgrounded server and reap it; MSYS cannot signal a native python.exe,
|
||||
# so only -9 lands. Every step is "|| true": callers run under set -e, and reaping
|
||||
# a server we just signalled makes wait return 143.
|
||||
stop_server() {
|
||||
test -n "${1:-}" || return 0
|
||||
kill "$1" 2>/dev/null || true
|
||||
if is_windows; then kill_tree "$1"; fi
|
||||
if is_windows; then
|
||||
kill -9 "$1" 2>/dev/null || true
|
||||
fi
|
||||
wait "$1" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
@@ -107,19 +109,3 @@ run_with_timeout() {
|
||||
done
|
||||
wait "$pid"
|
||||
}
|
||||
|
||||
# Bound an already-backgrounded crawl (pid $1) at $2s, reaping it and returning 124
|
||||
# on overrun: a wedge past --max-time would else block wait() forever and hang the CI step.
|
||||
wait_bounded() {
|
||||
local pid=$1 secs=$2 waited=0
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
if test "$waited" -ge "$secs"; then
|
||||
kill_tree "$pid"
|
||||
wait "$pid" 2>/dev/null || true
|
||||
return 124
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
wait "$pid"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user